From dd49220902635cf5d48955525e6972966d03a059 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 4 Nov 2026 14:38:03 +0100 Subject: [PATCH 1/6] feat: Collect the containerdebug logs --- CHANGELOG.md | 7 + .../opensearch/pages/usage-guide/logging.adoc | 8 + .../src/controller/build/node_config.rs | 29 +- .../build/product_logging/vector-test.yaml | 278 ++++++++++++++++++ .../build/product_logging/vector.yaml | 86 ++++++ .../controller/build/role_group_builder.rs | 24 +- ...pensearch-vector-aggregator-values.yaml.j2 | 18 ++ tests/templates/kuttl/smoke/10-assert.yaml.j2 | 6 +- 8 files changed, 451 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21cf5ac2..5e7c4a3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ All notable changes to this project will be documented in this file. ([#186]). - Add support for OpenSearch 3.8.0, deprecate 3.6.0 and remove support for 3.4.0 ([#189]). - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#190]). +- The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to + the Vector aggregator. The operator enables the file logs of containerdebug for this, with an + hourly rotation that keeps at most two files ([#205]). ### Changed @@ -43,6 +46,9 @@ All notable changes to this project will be documented in this file. - Fix a longstanding problem of including empty `categories`, `shortNames` and `additionalPrinterColumns` in the CRDs, which could cause problems with GitOps tools (e.g. ArgoCD) reporting a diff in the custom resources. See [our internal issue](https://github.com/stackabletech/hdfs-operator/issues/626) and [the fix](https://github.com/kube-rs/kube/pull/2042) for details ([#172]). +- The `containerdebug` process is started again. It was guarded with `command --search`, which is + fish syntax and fails in bash, so the guard was always false and containerdebug never ran + ([#205]). [#166]: https://github.com/stackabletech/opensearch-operator/pull/166 [#171]: https://github.com/stackabletech/opensearch-operator/pull/171 @@ -57,6 +63,7 @@ All notable changes to this project will be documented in this file. [#199]: https://github.com/stackabletech/opensearch-operator/pull/199 [#201]: https://github.com/stackabletech/opensearch-operator/pull/201 [#202]: https://github.com/stackabletech/opensearch-operator/pull/202 +[#205]: https://github.com/stackabletech/opensearch-operator/pull/205 ## [26.7.0] - 2026-07-21 diff --git a/docs/modules/opensearch/pages/usage-guide/logging.adoc b/docs/modules/opensearch/pages/usage-guide/logging.adoc index d9d45949..66fee5fe 100644 --- a/docs/modules/opensearch/pages/usage-guide/logging.adoc +++ b/docs/modules/opensearch/pages/usage-guide/logging.adoc @@ -31,4 +31,12 @@ spec: level: INFO ---- +NOTE: The logs of the xref:concepts:observability/containerdebug.adoc[`containerdebug` process] are collected as well. +They are labelled `container: containerdebug`, although no such Kubernetes container exists. +The process runs in the background of the `opensearch` container. +The label comes from `/stackable/log/containerdebug`, the directory which the operator sets with the `FILE_LOG_DIRECTORY` environment variable and from which the Vector agent derives the container name. +Overriding `FILE_LOG_DIRECTORY` with `envOverrides` therefore changes how these logs are aggregated. +The agent only collects files one level below `/stackable/log`, so a sibling directory such as `/stackable/log/opensearch` relabels the events instead of dropping them, while any deeper or shallower path stops the aggregation. +A path outside `/stackable/log` also leaves the size-capped log volume. + Further information on how to configure logging, can be found in xref:concepts:logging.adoc[]. diff --git a/rust/operator-binary/src/controller/build/node_config.rs b/rust/operator-binary/src/controller/build/node_config.rs index 6f202a4c..1b1e53b0 100644 --- a/rust/operator-binary/src/controller/build/node_config.rs +++ b/rust/operator-binary/src/controller/build/node_config.rs @@ -144,6 +144,9 @@ const CONFIG_OPTION_TRANSPORT_PUBLISH_HOST: &str = "transport.publish_host"; const DEFAULT_OPENSEARCH_HOME: &str = "/stackable/opensearch"; constant!(ENV_VAR_NAME_DISCOVERY_SEED_HOSTS: EnvVarName = CONFIG_OPTION_DISCOVERY_SEED_HOSTS); +constant!(ENV_VAR_NAME_FILE_LOG_DIRECTORY: EnvVarName = "FILE_LOG_DIRECTORY"); +constant!(ENV_VAR_NAME_FILE_LOG_MAX_FILES: EnvVarName = "FILE_LOG_MAX_FILES"); +constant!(ENV_VAR_NAME_FILE_LOG_ROTATION_PERIOD: EnvVarName = "FILE_LOG_ROTATION_PERIOD"); constant!(ENV_VAR_NAME_HTTP_PUBLISH_HOST: EnvVarName = CONFIG_OPTION_HTTP_PUBLISH_HOST); constant!(ENV_VAR_NAME_INITIAL_CLUSTER_MANAGER_NODES: EnvVarName = CONFIG_OPTION_INITIAL_CLUSTER_MANAGER_NODES); constant!(ENV_VAR_NAME_NETWORK_PUBLISH_HOST: EnvVarName = CONFIG_OPTION_NETWORK_PUBLISH_HOST); @@ -381,7 +384,19 @@ impl NodeConfig { .collect::>(), ) .expect("Node roles cannot contain commas, therefore creating a comma-separated list is safe."), - ); + ) + // Let the `containerdebug` process write its tracing information to a file. These + // are the generic `stackable-telemetry` variables and apply to the whole OpenSearch + // container, so any other telemetry-based process in it would log into the same + // directory and the Vector agent would label its events with + // `container: containerdebug`. + .with_value( + &ENV_VAR_NAME_FILE_LOG_DIRECTORY, + format!("{STACKABLE_LOG_DIR}/containerdebug"), + ) + // Rotation is opt-in, so without these the file would grow unbounded. + .with_value(&ENV_VAR_NAME_FILE_LOG_ROTATION_PERIOD, "hourly") + .with_value(&ENV_VAR_NAME_FILE_LOG_MAX_FILES, "2"); if let Some(initial_cluster_manager_nodes) = self.initial_cluster_manager_nodes() { env_vars = env_vars.with_value( @@ -725,6 +740,9 @@ mod tests { fn test_constants() { // Test that dereferencing the constants does not panic. let _ = *ENV_VAR_NAME_DISCOVERY_SEED_HOSTS; + let _ = *ENV_VAR_NAME_FILE_LOG_DIRECTORY; + let _ = *ENV_VAR_NAME_FILE_LOG_MAX_FILES; + let _ = *ENV_VAR_NAME_FILE_LOG_ROTATION_PERIOD; let _ = *ENV_VAR_NAME_HTTP_PUBLISH_HOST; let _ = *ENV_VAR_NAME_INITIAL_CLUSTER_MANAGER_NODES; let _ = *ENV_VAR_NAME_NETWORK_PUBLISH_HOST; @@ -763,6 +781,15 @@ mod tests { assert_eq!( EnvVarSet::new() + .with_value( + &EnvVarName::from_str_unsafe("FILE_LOG_DIRECTORY"), + "/stackable/log/containerdebug", + ) + .with_value(&EnvVarName::from_str_unsafe("FILE_LOG_MAX_FILES"), "2") + .with_value( + &EnvVarName::from_str_unsafe("FILE_LOG_ROTATION_PERIOD"), + "hourly", + ) .with_value(&EnvVarName::from_str_unsafe("TEST"), "value") .with_field_path( &EnvVarName::from_str_unsafe("_POD_NAME"), diff --git a/rust/operator-binary/src/controller/build/product_logging/vector-test.yaml b/rust/operator-binary/src/controller/build/product_logging/vector-test.yaml index 75511d22..8a8a44c7 100644 --- a/rust/operator-binary/src/controller/build/product_logging/vector-test.yaml +++ b/rust/operator-binary/src/controller/build/product_logging/vector-test.yaml @@ -439,6 +439,284 @@ tests: "timestamp": t'2025-10-01T12:47:28.582+0000' } + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":"found disk","disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331+0000 + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "found disk\n\ndisk.name=overlay", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager" + } + + assert_eq!(expected_log_event, .) + - name: Test unparsable tracing-rs log + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{ unparsable' + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331+0000 + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "containerdebug", + "errors": [ + .errors[0] + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "{ unparsable", + "timestamp": "2026-11-02T14:49:29.473487331+0000", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager" + } + + assert_eq!(expected_log_event, .) + + # The remainder of the message contains character offsets into + # the VRL source, so only the stable prefix is asserted. + assert!(starts_with!(.errors[0], "JSON not parsable: ")) + - name: Test tracing-rs log which is not a JSON object + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: 'false' + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331+0000 + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "containerdebug", + "errors": [ + "Parsed event is not a JSON object." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "false", + "timestamp": "2026-11-02T14:49:29.473487331+0000", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager" + } + + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log with invalid fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"unparsable","level":"INVALID","fields":"none","target":null}' + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331+0000 + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "containerdebug", + "errors": [ + .errors[0], + "Logger/target not found.", + "Level \"INVALID\" unknown, using \"INFO\" instead.", + "Fields are not an object." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "{\"timestamp\":\"unparsable\",\"level\":\"INVALID\",\"fields\":\"none\",\"target\":null}", + "timestamp": "2026-11-02T14:49:29.473487331+0000", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager" + } + + assert_eq!(expected_log_event, .) + + # The remainder of the message contains character offsets into + # the VRL source, so only the stable prefix is asserted. + assert!(starts_with!(.errors[0], "Timestamp not parsable, using current time instead: ")) + - name: Test tracing-rs log without a timestamp + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"level":"INFO","fields":{"message":"found disk"},"target":"containerdebug::system_information::disk"}' + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331+0000 + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "containerdebug", + "errors": [ + "Timestamp not found, using current time instead." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "found disk", + "timestamp": "2026-11-02T14:49:29.473487331+0000", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager" + } + + assert_eq!(expected_log_event, .) + + # The timestamp which the file source attached to the event is kept, + # as in the opensearch_server transform. + assert!(is_string(.timestamp)) + - name: Test tracing-rs log without a message but with other fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331+0000 + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "disk.name=overlay", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager" + } + + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log with a message which is not a string + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":42,"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331+0000 + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "disk.name=overlay\nmessage=42", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager" + } + + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log without any fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{},"target":"containerdebug::system_information::disk"}' + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331+0000 + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "containerdebug", + "errors": [ + "Message not found." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager" + } + assert_eq!(expected_log_event, .) - name: Test Vector internal logs inputs: diff --git a/rust/operator-binary/src/controller/build/product_logging/vector.yaml b/rust/operator-binary/src/controller/build/product_logging/vector.yaml index 766ea3a8..bb5f2004 100644 --- a/rust/operator-binary/src/controller/build/product_logging/vector.yaml +++ b/rust/operator-binary/src/controller/build/product_logging/vector.yaml @@ -16,6 +16,12 @@ sources: start_pattern: ^\{ timeout_ms: 100 + # Reads the logs of the containerdebug process + files_tracing_rs: + type: file + include: + - ${LOG_DIR}/*/*.tracing-rs.json + # Reads the internal Vector logs vector: type: internal_logs @@ -90,10 +96,90 @@ transforms: .message = join!(compact([.message, stacktrace]), "\n\n") } + # Transforms the tracing-rs JSON logs written by the containerdebug process + processed_files_tracing_rs: + inputs: + - files_tracing_rs + type: remap + source: | + raw_message = string!(.message) + + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + parsed_event, err = parse_json(raw_message) + if err != null { + error = "JSON not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else if !is_object(parsed_event) { + error = "Parsed event is not a JSON object." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event) + + timestamp_string, err = string(event.timestamp) + if err == null { + parsed_timestamp, err = parse_timestamp(timestamp_string, "%+") + if err == null { + .timestamp = parsed_timestamp + } else { + .errors = push(.errors, "Timestamp not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "Timestamp not found, using current time instead.") + } + + .logger, err = string(event.target) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger/target not found.") + } + + level, err = string(event.level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], upcase(level)) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = upcase(level) + } + + fields, err = object(event.fields) + if err == null { + message, err = string(fields.message) + if err == null { + # The message is prepended to the other fields, so remove it from + # them. + del(fields.message) + } else { + # Keep an unusable message among the other fields instead of + # discarding its content. + message = "" + } + + other_fields = encode_key_value(fields, field_delimiter: "\n") + .message = join!(compact([message, other_fields]), "\n\n") + } else { + .errors = push(.errors, "Fields are not an object.") + # Fall back to the raw line, so that its content is not lost. + .message = raw_message + } + + if is_empty(.message) { + .errors = push(.errors, "Message not found.") + } + } + # Extends the processed files with the fields "container" and "file" extended_logs_files: inputs: - processed_files_opensearch_server + - processed_files_tracing_rs type: remap source: | del(.source_type) diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index 76190d57..85d10684 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -767,7 +767,8 @@ impl<'a> RoleGroupBuilder<'a> { "{COMMON_BASH_TRAP_FUNCTIONS}\n\ {remove_vector_shutdown_file_command}\n\ prepare_signal_handlers\n\ - if command --search containerdebug >/dev/null 2>&1; then\n\ + if command -v containerdebug >/dev/null 2>&1; then\n\ + mkdir --parents {STACKABLE_LOG_DIR}/containerdebug\n\ containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &\n\ else\n\ echo >&2 \"containerdebug not installed; Proceed without it.\"\n\ @@ -1016,6 +1017,12 @@ impl<'a> RoleGroupBuilder<'a> { Volume { name: LOG_VOLUME_NAME.to_string(), empty_dir: Some(EmptyDirVolumeSource { + // Only the OpenSearch log files are budgeted here. The `containerdebug` + // files in this volume are rotated by time, which bounds their number but + // not their size, so they are an unbudgeted writer. They stay negligible at + // the default INFO level and a 30 minute loop, but raising `FILE_LOG_LEVEL` + // via `envOverrides` can exceed the limit, which makes the kubelet evict the + // Pod. size_limit: Some(calculate_log_volume_size_limit(&[ MAX_OPENSEARCH_SERVER_LOG_FILES_SIZE, ])), @@ -2076,7 +2083,8 @@ mod tests { "\n", "rm -f /stackable/log/_vector/shutdown\n", "prepare_signal_handlers\n", - "if command --search containerdebug >/dev/null 2>&1; then\n", + "if command -v containerdebug >/dev/null 2>&1; then\n", + "mkdir --parents /stackable/log/containerdebug\n", "containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n", "else\n", "echo >&2 \"containerdebug not installed; Proceed without it.\"\n", @@ -2094,6 +2102,18 @@ mod tests { "-c" ], "env": [ + { + "name": "FILE_LOG_DIRECTORY", + "value": "/stackable/log/containerdebug" + }, + { + "name": "FILE_LOG_MAX_FILES", + "value": "2" + }, + { + "name": "FILE_LOG_ROTATION_PERIOD", + "value": "hourly" + }, { "name": "_POD_NAME", "valueFrom": { diff --git a/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 index f7782b02..2ba0b284 100644 --- a/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 +++ b/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 @@ -32,6 +32,13 @@ customConfig: condition: >- .pod == "opensearch-nodes-automatic-log-config-0" && .container == "opensearch" + filteredAutomaticLogConfigContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "opensearch-nodes-automatic-log-config-0" && + .container == "containerdebug" filteredAutomaticLogConfigVector: type: filter inputs: @@ -46,6 +53,13 @@ customConfig: condition: >- .pod == "opensearch-nodes-custom-log-config-0" && .container == "opensearch" + filteredCustomLogConfigContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "opensearch-nodes-custom-log-config-0" && + .container == "containerdebug" filteredCustomLogConfigVector: type: filter inputs: @@ -73,6 +87,10 @@ customConfig: type: console encoding: codec: json + buffer: + # Avoid back pressure from a stalling stdout. The test should not + # fail because of a sink which only aids debugging. + when_full: drop_newest {% if lookup('env', 'VECTOR_AGGREGATOR') %} aggregator: inputs: diff --git a/tests/templates/kuttl/smoke/10-assert.yaml.j2 b/tests/templates/kuttl/smoke/10-assert.yaml.j2 index e2b6de55..86145ff3 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-assert.yaml.j2 @@ -99,7 +99,8 @@ spec: rm -f /stackable/log/_vector/shutdown prepare_signal_handlers - if command --search containerdebug >/dev/null 2>&1; then + if command -v containerdebug >/dev/null 2>&1; then + mkdir --parents /stackable/log/containerdebug containerdebug --output=/stackable/log/containerdebug-state.json --loop & else echo >&2 "containerdebug not installed; Proceed without it." @@ -585,7 +586,8 @@ spec: rm -f /stackable/log/_vector/shutdown prepare_signal_handlers - if command --search containerdebug >/dev/null 2>&1; then + if command -v containerdebug >/dev/null 2>&1; then + mkdir --parents /stackable/log/containerdebug containerdebug --output=/stackable/log/containerdebug-state.json --loop & else echo >&2 "containerdebug not installed; Proceed without it." From 5c018d7d44694becd3f4957c1d1f408e9685337d Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 4 Nov 2026 15:26:53 +0100 Subject: [PATCH 2/6] test(logging): Comment the stdout sink --- .../logging/10_opensearch-vector-aggregator-values.yaml.j2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 index 2ba0b284..2226f28b 100644 --- a/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 +++ b/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 @@ -81,6 +81,12 @@ customConfig: inputs: - filtered* type: blackhole + # Shows all events received from the agents. This sink is intentional and + # not a debugging leftover: if a "filtered*" transform gathers no event, + # then the assertion only reports a count, whereas this log shows which + # pod and container names actually arrived. A failure in CI cannot be + # reproduced by enabling this afterwards, so it is kept enabled at the + # price of the log volume. stdout: inputs: - vector From 7e6cfa6a29fb4ce6ebe82f8b35a689300320a4f8 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 4 Nov 2026 16:10:14 +0100 Subject: [PATCH 3/6] refactor: Introduce a constant for the containerdebug log directory --- Cargo.lock | 1 + Cargo.nix | 4 ++++ Cargo.toml | 1 + rust/operator-binary/Cargo.toml | 1 + rust/operator-binary/src/controller/build.rs | 7 +++++++ rust/operator-binary/src/controller/build/node_config.rs | 4 ++-- .../src/controller/build/role_group_builder.rs | 3 ++- 7 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 48c1a6c9..5d88af08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3177,6 +3177,7 @@ version = "0.0.0-dev" dependencies = [ "built", "clap", + "const_format", "futures 0.3.34", "pretty_assertions", "regex", diff --git a/Cargo.nix b/Cargo.nix index c99db4f0..b8c56950 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -10486,6 +10486,10 @@ rec { name = "clap"; packageId = "clap"; } + { + name = "const_format"; + packageId = "const_format"; + } { name = "futures"; packageId = "futures 0.3.34"; diff --git a/Cargo.toml b/Cargo.toml index afa584e8..50f523a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", built = { version = "0.8", features = ["chrono", "git2"] } clap = "4.6" +const_format = "0.2" futures = { version = "0.3", features = ["compat"] } pretty_assertions = "1.4" regex = "1.13" diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index d0114a67..3047fcb5 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -19,6 +19,7 @@ test-support = [] stackable-operator.workspace = true clap.workspace = true +const_format.workspace = true futures.workspace = true regex.workspace = true semver.workspace = true diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index 27278cba..d6fc77da 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -2,7 +2,9 @@ use std::marker::PhantomData; +use const_format::concatcp; use role_builder::RoleBuilder; +use stackable_operator::v2::product_logging::framework::STACKABLE_LOG_DIR; use super::{ContextNames, KubernetesResources, Prepared, ValidatedCluster}; @@ -11,6 +13,11 @@ pub mod product_logging; pub mod role_builder; pub mod role_group_builder; +/// Directory which the `containerdebug` process writes its log files to. It must be exactly one +/// level below the log directory, because the Vector agent derives the container name from that +/// path segment. +pub const CONTAINERDEBUG_LOG_DIR: &str = concatcp!(STACKABLE_LOG_DIR, "/containerdebug"); + /// Builds Kubernetes resource specifications from the given validated cluster /// /// This function cannot fail because all failing conditions were already checked in the validation diff --git a/rust/operator-binary/src/controller/build/node_config.rs b/rust/operator-binary/src/controller/build/node_config.rs index 1b1e53b0..2f4c4113 100644 --- a/rust/operator-binary/src/controller/build/node_config.rs +++ b/rust/operator-binary/src/controller/build/node_config.rs @@ -17,7 +17,7 @@ use stackable_operator::{ }, }; -use super::ValidatedCluster; +use super::{CONTAINERDEBUG_LOG_DIR, ValidatedCluster}; use crate::{ controller::{ NODES_ROLE_NAME, OpenSearchRoleGroupConfig, ValidatedNodeRole, @@ -392,7 +392,7 @@ impl NodeConfig { // `container: containerdebug`. .with_value( &ENV_VAR_NAME_FILE_LOG_DIRECTORY, - format!("{STACKABLE_LOG_DIR}/containerdebug"), + CONTAINERDEBUG_LOG_DIR, ) // Rotation is opt-in, so without these the file would grow unbounded. .with_value(&ENV_VAR_NAME_FILE_LOG_ROTATION_PERIOD, "hourly") diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index 85d10684..fa86d188 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -74,6 +74,7 @@ use stackable_operator::{ }; use super::{ + CONTAINERDEBUG_LOG_DIR, node_config::{CONFIGURATION_FILE_OPENSEARCH_YML, NodeConfig}, product_logging::config::{ CONFIGURATION_FILE_LOG4J2_PROPERTIES, create_log4j2_config, vector_config_file_content, @@ -768,7 +769,7 @@ impl<'a> RoleGroupBuilder<'a> { {remove_vector_shutdown_file_command}\n\ prepare_signal_handlers\n\ if command -v containerdebug >/dev/null 2>&1; then\n\ - mkdir --parents {STACKABLE_LOG_DIR}/containerdebug\n\ + mkdir --parents {CONTAINERDEBUG_LOG_DIR}\n\ containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &\n\ else\n\ echo >&2 \"containerdebug not installed; Proceed without it.\"\n\ From 81c47cb5dc843e8fa8a1fc69d1a441a8091f9f41 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 4 Nov 2026 16:57:42 +0100 Subject: [PATCH 4/6] chore: Update comment --- .../src/controller/build/role_group_builder.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index fa86d188..95e4af2c 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -1018,12 +1018,9 @@ impl<'a> RoleGroupBuilder<'a> { Volume { name: LOG_VOLUME_NAME.to_string(), empty_dir: Some(EmptyDirVolumeSource { - // Only the OpenSearch log files are budgeted here. The `containerdebug` - // files in this volume are rotated by time, which bounds their number but - // not their size, so they are an unbudgeted writer. They stay negligible at - // the default INFO level and a 30 minute loop, but raising `FILE_LOG_LEVEL` - // via `envOverrides` can exceed the limit, which makes the kubelet evict the - // Pod. + // Only the OpenSearch log files are budgeted. The `containerdebug` files in this + // volume are rotated by time, not by size, so they have no budget; raising + // `FILE_LOG_LEVEL` via `envOverrides` can therefore evict the Pod. size_limit: Some(calculate_log_volume_size_limit(&[ MAX_OPENSEARCH_SERVER_LOG_FILES_SIZE, ])), From c6633e3740619c3500d6032a11b233b352c542c2 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Thu, 5 Nov 2026 11:38:59 +0100 Subject: [PATCH 5/6] chore: Follow-ups to the containerdebug log collection --- rust/operator-binary/src/controller/build.rs | 7 ------- .../src/controller/build/node_config.rs | 5 +++-- .../src/controller/build/role_group_builder.rs | 5 ++--- tests/templates/kuttl/smoke/10-assert.yaml.j2 | 16 ++++++++++++++-- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index d6fc77da..27278cba 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -2,9 +2,7 @@ use std::marker::PhantomData; -use const_format::concatcp; use role_builder::RoleBuilder; -use stackable_operator::v2::product_logging::framework::STACKABLE_LOG_DIR; use super::{ContextNames, KubernetesResources, Prepared, ValidatedCluster}; @@ -13,11 +11,6 @@ pub mod product_logging; pub mod role_builder; pub mod role_group_builder; -/// Directory which the `containerdebug` process writes its log files to. It must be exactly one -/// level below the log directory, because the Vector agent derives the container name from that -/// path segment. -pub const CONTAINERDEBUG_LOG_DIR: &str = concatcp!(STACKABLE_LOG_DIR, "/containerdebug"); - /// Builds Kubernetes resource specifications from the given validated cluster /// /// This function cannot fail because all failing conditions were already checked in the validation diff --git a/rust/operator-binary/src/controller/build/node_config.rs b/rust/operator-binary/src/controller/build/node_config.rs index 2f4c4113..d1056ba4 100644 --- a/rust/operator-binary/src/controller/build/node_config.rs +++ b/rust/operator-binary/src/controller/build/node_config.rs @@ -2,6 +2,7 @@ use std::str::FromStr; +use const_format::concatcp; use serde_json::json; use stackable_operator::{ builder::pod::container::FieldPathEnvVar, @@ -17,7 +18,7 @@ use stackable_operator::{ }, }; -use super::{CONTAINERDEBUG_LOG_DIR, ValidatedCluster}; +use super::ValidatedCluster; use crate::{ controller::{ NODES_ROLE_NAME, OpenSearchRoleGroupConfig, ValidatedNodeRole, @@ -392,7 +393,7 @@ impl NodeConfig { // `container: containerdebug`. .with_value( &ENV_VAR_NAME_FILE_LOG_DIRECTORY, - CONTAINERDEBUG_LOG_DIR, + concatcp!(STACKABLE_LOG_DIR, "/containerdebug"), ) // Rotation is opt-in, so without these the file would grow unbounded. .with_value(&ENV_VAR_NAME_FILE_LOG_ROTATION_PERIOD, "hourly") diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index 95e4af2c..1eab92bc 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -74,7 +74,6 @@ use stackable_operator::{ }; use super::{ - CONTAINERDEBUG_LOG_DIR, node_config::{CONFIGURATION_FILE_OPENSEARCH_YML, NodeConfig}, product_logging::config::{ CONFIGURATION_FILE_LOG4J2_PROPERTIES, create_log4j2_config, vector_config_file_content, @@ -769,7 +768,7 @@ impl<'a> RoleGroupBuilder<'a> { {remove_vector_shutdown_file_command}\n\ prepare_signal_handlers\n\ if command -v containerdebug >/dev/null 2>&1; then\n\ - mkdir --parents {CONTAINERDEBUG_LOG_DIR}\n\ + mkdir --parents \"$FILE_LOG_DIRECTORY\"\n\ containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &\n\ else\n\ echo >&2 \"containerdebug not installed; Proceed without it.\"\n\ @@ -2082,7 +2081,7 @@ mod tests { "rm -f /stackable/log/_vector/shutdown\n", "prepare_signal_handlers\n", "if command -v containerdebug >/dev/null 2>&1; then\n", - "mkdir --parents /stackable/log/containerdebug\n", + "mkdir --parents \"$FILE_LOG_DIRECTORY\"\n", "containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n", "else\n", "echo >&2 \"containerdebug not installed; Proceed without it.\"\n", diff --git a/tests/templates/kuttl/smoke/10-assert.yaml.j2 b/tests/templates/kuttl/smoke/10-assert.yaml.j2 index 86145ff3..920f3a06 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-assert.yaml.j2 @@ -100,7 +100,7 @@ spec: rm -f /stackable/log/_vector/shutdown prepare_signal_handlers if command -v containerdebug >/dev/null 2>&1; then - mkdir --parents /stackable/log/containerdebug + mkdir --parents "$FILE_LOG_DIRECTORY" containerdebug --output=/stackable/log/containerdebug-state.json --loop & else echo >&2 "containerdebug not installed; Proceed without it." @@ -117,6 +117,12 @@ spec: env: - name: DISABLE_INSTALL_DEMO_CONFIG value: "true" + - name: FILE_LOG_DIRECTORY + value: /stackable/log/containerdebug + - name: FILE_LOG_MAX_FILES + value: "2" + - name: FILE_LOG_ROTATION_PERIOD + value: hourly - name: OPENSEARCH_HOME value: {{ test_scenario['values']['opensearch_home'] }} - name: _POD_NAME @@ -587,7 +593,7 @@ spec: rm -f /stackable/log/_vector/shutdown prepare_signal_handlers if command -v containerdebug >/dev/null 2>&1; then - mkdir --parents /stackable/log/containerdebug + mkdir --parents "$FILE_LOG_DIRECTORY" containerdebug --output=/stackable/log/containerdebug-state.json --loop & else echo >&2 "containerdebug not installed; Proceed without it." @@ -604,6 +610,12 @@ spec: env: - name: DISABLE_INSTALL_DEMO_CONFIG value: "true" + - name: FILE_LOG_DIRECTORY + value: /stackable/log/containerdebug + - name: FILE_LOG_MAX_FILES + value: "2" + - name: FILE_LOG_ROTATION_PERIOD + value: hourly - name: OPENSEARCH_HOME value: {{ test_scenario['values']['opensearch_home'] }} - name: _POD_NAME From ee3ea430afaa791fd35eb1052f36ba3aa32924ce Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Fri, 6 Nov 2026 09:18:56 +0100 Subject: [PATCH 6/6] chore: Upgrade serde-saphyr --- Cargo.lock | 6 +++--- Cargo.nix | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d88af08..1d44316f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2884,12 +2884,12 @@ dependencies = [ [[package]] name = "serde-saphyr" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f4dc2714fc5632417a6e979602d7044bad8f2880c63fc7217087180acec31b6" +checksum = "d43fb82d550b07e6f929a58d7d0935182602f48ddead875f5ee313c41c8f07fe" dependencies = [ "annotate-snippets", - "base64 0.23.1", + "base64 0.22.1", "encoding_rs_io", "granit-parser", "nohash-hasher", diff --git a/Cargo.nix b/Cargo.nix index b8c56950..c5e7959b 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -9578,10 +9578,10 @@ rec { }; "serde-saphyr" = rec { crateName = "serde-saphyr"; - version = "2.1.0"; + version = "2.1.1"; edition = "2024"; crateBin = []; - sha256 = "1diixjn80w882xrgqqqci3rdifj4f0nn0yg9lqbj8qy59xqw4k9z"; + sha256 = "1zh7iwfc84z3brgqgbfyips049hq6l4pv3d557wyc1qbalnvhgyl"; libName = "serde_saphyr"; dependencies = [ { @@ -9591,7 +9591,7 @@ rec { } { name = "base64"; - packageId = "base64 0.23.1"; + packageId = "base64 0.22.1"; optional = true; } {