diff --git a/CHANGELOG.md b/CHANGELOG.md index 845a1cf6..76241f67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ All notable changes to this project will be documented in this file. ([#831]). - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#835]). - Name nodes now have a default affinity to the OPA Pods when OPA authorization is configured ([#839]). +- The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to + the Vector aggregator ([#857]). ### Changed @@ -62,6 +64,10 @@ All notable changes to this project will be documented in this file. replicas, and never for datanodes ([#830]). - Set `dfs.encrypt.data.transfer.cipher.suites` instead of the misspelt `dfs.encrypt.data.transfer.cipher.suite`. Hadoop ignored the misspelt key, so block data was never encrypted with AES/CTR and was only protected by SASL DIGEST-MD5 ([#854]). +- The `containerdebug` process writes its log file again. The operator set + `CONTAINERDEBUG_LOG_DIRECTORY`, which containerdebug stopped reading in its version 0.2.0 when it + moved to `stackable-telemetry`; the directory is now set via `FILE_LOG_DIRECTORY`. The file is + rotated hourly and at most two files are kept, because rotation is opt-in ([#857]). ### Removed @@ -89,6 +95,7 @@ All notable changes to this project will be documented in this file. [#848]: https://github.com/stackabletech/hdfs-operator/pull/848 [#850]: https://github.com/stackabletech/hdfs-operator/pull/850 [#854]: https://github.com/stackabletech/hdfs-operator/pull/854 +[#857]: https://github.com/stackabletech/hdfs-operator/pull/857 ## [26.7.0] - 2026-07-21 diff --git a/docs/modules/hdfs/pages/usage-guide/logging-log-aggregation.adoc b/docs/modules/hdfs/pages/usage-guide/logging-log-aggregation.adoc index c4c3c208..c3f4dd34 100644 --- a/docs/modules/hdfs/pages/usage-guide/logging-log-aggregation.adoc +++ b/docs/modules/hdfs/pages/usage-guide/logging-log-aggregation.adoc @@ -22,4 +22,12 @@ spec: enableVectorAgent: true ---- +NOTE: The logs of the xref:concepts:observability/containerdebug.adoc[`containerdebug` process] are collected as well. +They are labelled `container: containerdebug`, although no such Kubernetes container exists. +The process runs in the background of the `hdfs` container. +The label comes from `/stackable/log/containerdebug`, the directory which the operator sets with the `FILE_LOG_DIRECTORY` environment variable and from which the Vector agent derives the container name. +Overriding `FILE_LOG_DIRECTORY` with `envOverrides` therefore changes how these logs are aggregated. +The agent only collects files one level below `/stackable/log`, so a sibling directory such as `/stackable/log/hdfs` relabels the events instead of dropping them, while any deeper or shallower path stops the aggregation. +A path outside `/stackable/log` also leaves the size-capped log volume. + Further information on how to configure logging, can be found in xref:concepts:logging.adoc[]. diff --git a/rust/operator-binary/src/controller/build/container.rs b/rust/operator-binary/src/controller/build/container.rs index 03586da6..379952c7 100644 --- a/rust/operator-binary/src/controller/build/container.rs +++ b/rust/operator-binary/src/controller/build/container.rs @@ -11,6 +11,7 @@ //! use std::{collections::BTreeMap, str::FromStr}; +use const_format::concatcp; use indoc::formatdoc; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ @@ -115,7 +116,9 @@ const ENV_TOPOLOGY_LABELS: &str = "TOPOLOGY_LABELS"; const ENV_HADOOP_OPTS: &str = "HADOOP_OPTS"; const ENV_KRB5_CONFIG: &str = "KRB5_CONFIG"; const ENV_KRB5_CLIENT_KTNAME: &str = "KRB5_CLIENT_KTNAME"; -const ENV_CONTAINERDEBUG_LOG_DIRECTORY: &str = "CONTAINERDEBUG_LOG_DIRECTORY"; +const ENV_FILE_LOG_DIRECTORY: &str = "FILE_LOG_DIRECTORY"; +const ENV_FILE_LOG_MAX_FILES: &str = "FILE_LOG_MAX_FILES"; +const ENV_FILE_LOG_ROTATION_PERIOD: &str = "FILE_LOG_ROTATION_PERIOD"; type Result = std::result::Result; @@ -570,6 +573,7 @@ impl ContainerConfig { {COMMON_BASH_TRAP_FUNCTIONS} {remove_vector_shutdown_file_command} prepare_signal_handlers + mkdir --parents "$FILE_LOG_DIRECTORY" containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & if [[ -d {LISTENER_VOLUME_DIR} ]]; then export POD_ADDRESS=$(cat {LISTENER_VOLUME_DIR}/default-address/address) @@ -859,15 +863,34 @@ impl ContainerConfig { ); } - // Needed for the `containerdebug` process to log it's tracing information to. - env.insert( - ENV_CONTAINERDEBUG_LOG_DIRECTORY.to_string(), - EnvVar { - name: ENV_CONTAINERDEBUG_LOG_DIRECTORY.to_string(), - value: Some(format!("{STACKABLE_LOG_DIR}/containerdebug")), - value_from: None, - }, - ); + // Let the `containerdebug` process write its tracing information to a file. These are the + // generic `stackable-telemetry` variables and apply to the whole container, so any other + // telemetry-based process in it would log into the same directory and the Vector agent + // would label its events with `container: containerdebug`. + // Rotation is opt-in, so without the last two the file would grow unbounded. + // Only the main container runs `containerdebug`, so the variables would have no effect + // anywhere else. + if let ContainerConfig::Hdfs { .. } = self { + for (name, value) in [ + // The directory must be exactly one level below the log directory, because the + // Vector agent derives the container name from that path segment. + ( + ENV_FILE_LOG_DIRECTORY, + concatcp!(STACKABLE_LOG_DIR, "/containerdebug"), + ), + (ENV_FILE_LOG_ROTATION_PERIOD, "hourly"), + (ENV_FILE_LOG_MAX_FILES, "2"), + ] { + env.insert( + name.to_owned(), + EnvVar { + name: name.to_owned(), + value: Some(value.to_owned()), + ..EnvVar::default() + }, + ); + } + } // Overrides need to come last let mut env_override_vars: BTreeMap = inputs @@ -973,6 +996,10 @@ impl ContainerConfig { VolumeBuilder::new(ContainerConfig::STACKABLE_LOG_VOLUME_MOUNT_NAME) .empty_dir(EmptyDirVolumeSource { medium: None, + // Only the log files listed here are budgeted. The `containerdebug` + // files in this volume are rotated by time, not by size, so they have + // no budget; raising `FILE_LOG_LEVEL` via `envOverrides` can therefore + // evict the Pod. size_limit: Some( product_logging::framework::calculate_log_volume_size_limit(&[ MAX_HDFS_LOG_FILE_SIZE, diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs index e20a7046..58e13e4f 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs @@ -176,5 +176,7 @@ mod tests { assert!(!content.is_empty()); // HDFS containers log via log4j to `*.log4j.xml`, so the `files_log4j` source matches them. assert!(content.contains("files_log4j")); + assert!(content.contains("files_tracing_rs")); + assert!(content.contains("*.tracing-rs.json")); } } diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml index 379f4cc3..6fdd4bf5 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml @@ -137,6 +137,285 @@ tests: "timestamp": t'2025-10-02T09:27:28.582Z' } + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":"found disk","disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: test-hdfs-namenode-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "test-hdfs", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "found disk\n\ndisk.name=overlay", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "test-hdfs-namenode-default-0", + "role": "namenode", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Test unparsable tracing-rs log + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{ unparsable' + pod: test-hdfs-namenode-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "test-hdfs", + "container": "containerdebug", + "errors": [ + .errors[0] + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "{ unparsable", + "timestamp": .timestamp, + "namespace": "default", + "pod": "test-hdfs-namenode-default-0", + "role": "namenode", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The remainder of the message contains character offsets into + # the VRL source, so only the stable prefix is asserted. + assert!(starts_with!(.errors[0], "JSON not parsable: ")) + - name: Test tracing-rs log which is not a JSON object + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: 'false' + pod: test-hdfs-namenode-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "test-hdfs", + "container": "containerdebug", + "errors": [ + "Parsed event is not a JSON object." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "false", + "timestamp": .timestamp, + "namespace": "default", + "pod": "test-hdfs-namenode-default-0", + "role": "namenode", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log with invalid fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"unparsable","level":"INVALID","fields":"none","target":null}' + pod: test-hdfs-namenode-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "test-hdfs", + "container": "containerdebug", + "errors": [ + .errors[0], + "Logger/target not found.", + "Level \"INVALID\" unknown, using \"INFO\" instead.", + "Fields are not an object." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "{\"timestamp\":\"unparsable\",\"level\":\"INVALID\",\"fields\":\"none\",\"target\":null}", + "timestamp": .timestamp, + "namespace": "default", + "pod": "test-hdfs-namenode-default-0", + "role": "namenode", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The remainder of the message contains character offsets into + # the VRL source, so only the stable prefix is asserted. + assert!(starts_with!(.errors[0], "Timestamp not parsable, using current time instead: ")) + - name: Test tracing-rs log without a timestamp + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"level":"INFO","fields":{"message":"found disk"},"target":"containerdebug::system_information::disk"}' + pod: test-hdfs-namenode-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "test-hdfs", + "container": "containerdebug", + "errors": [ + "Timestamp not found, using current time instead." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "found disk", + "timestamp": .timestamp, + "namespace": "default", + "pod": "test-hdfs-namenode-default-0", + "role": "namenode", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The current time is used and not the timestamp which the file + # source attached to the event. The latter reaches the transform + # as a string, whereas now() yields a timestamp. + assert!(is_timestamp(.timestamp)) + - name: Test tracing-rs log without a message but with other fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: test-hdfs-namenode-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "test-hdfs", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "disk.name=overlay", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "test-hdfs-namenode-default-0", + "role": "namenode", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log with a message which is not a string + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":42,"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: test-hdfs-namenode-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "test-hdfs", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "disk.name=overlay\nmessage=42", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "test-hdfs-namenode-default-0", + "role": "namenode", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log without any fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{},"target":"containerdebug::system_information::disk"}' + pod: test-hdfs-namenode-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "test-hdfs", + "container": "containerdebug", + "errors": [ + "Message not found." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "test-hdfs-namenode-default-0", + "role": "namenode", + "roleGroup": "default" + } + assert_eq!(expected_log_event, .) - name: Test Vector internal logs inputs: diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml index 1d4e04b0..fea1f94e 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml @@ -30,6 +30,12 @@ sources: condition_pattern: ^- .pod == "test-hdfs-automatic-log-namenode-default-0" && .container == "hdfs" filteredAutomaticLogConfigNameNode1: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-namenode-default-1" && .container == "hdfs" filteredAutomaticLogConfigNameNode0Zkfc: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-namenode-default-0" && .container == "zkfc" filteredAutomaticLogConfigNameNode1Zkfc: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-namenode-default-1" && .container == "zkfc" + filteredAutomaticLogConfigNameNode0Containerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-hdfs-automatic-log-namenode-default-0" && + .container == "containerdebug" filteredAutomaticLogConfigNameNode0Vector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-namenode-default-0" && .container == "vector" + filteredAutomaticLogConfigNameNode1Containerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-hdfs-automatic-log-namenode-default-1" && + .container == "containerdebug" filteredAutomaticLogConfigNameNode1Vector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-namenode-default-1" && .container == "vector" filteredAutomaticLogConfigNameNode0FormatNameNodeLog4j: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-namenode-default-0" && .container == "format-namenodes" && .file == "format-namenodes.log4j.xml" filteredAutomaticLogConfigNameNode0FormatNameNodeStdout: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-namenode-default-0" && .container == "format-namenodes" && .file == "container.stdout.log" filteredAutomaticLogConfigNameNode0FormatNameNodeStderr: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-namenode-default-0" && .container == "format-namenodes" && .file == "container.stderr.log" filteredAutomaticLogConfigNameNode1FormatNameNodeLog4j: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-namenode-default-1" && .container == "format-namenodes" && .file == "format-namenodes.log4j.xml" filteredAutomaticLogConfigNameNode1FormatNameNodeStdout: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-namenode-default-1" && .container == "format-namenodes" && .file == "container.stdout.log" filteredAutomaticLogConfigNameNode1FormatNameNodeStderr: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-namenode-default-1" && .container == "format-namenodes" && .file == "container.stderr.log" filteredAutomaticLogConfigNameNode0FormatZookeeperLog4j: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-namenode-default-0" && .container == "format-zookeeper" && .file == "format-zookeeper.log4j.xml" filteredAutomaticLogConfigNameNode0FormatZookeeperStdout: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-namenode-default-0" && .container == "format-zookeeper" && .file == "container.stdout.log" filteredAutomaticLogConfigNameNode0FormatZookeeperStderr: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-namenode-default-0" && .container == "format-zookeeper" && .file == "container.stderr.log" filteredAutomaticLogConfigNameNode1FormatZookeeperStdout: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-namenode-default-1" && .container == "format-zookeeper" && .file == "container.stdout.log" filteredAutomaticLogConfigNameNode1FormatZookeeperStderr: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-namenode-default-1" && .container == "format-zookeeper" && .file == "container.stderr.log" filteredAutomaticLogConfigDataNode0: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-datanode-default-0" && .container == "hdfs" + filteredAutomaticLogConfigDataNode0Containerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-hdfs-automatic-log-datanode-default-0" && + .container == "containerdebug" filteredAutomaticLogConfigDataNode0Vector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-datanode-default-0" && .container == "vector" filteredAutomaticLogConfigDataNode0WaitForNameNodesLog4j: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-datanode-default-0" && .container == "wait-for-namenodes" && .file == "wait-for-namenodes.log4j.xml" filteredAutomaticLogConfigDataNode0WaitForNameNodesStdout: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-datanode-default-0" && .container == "wait-for-namenodes" && .file == "container.stdout.log" filteredAutomaticLogConfigDataNode0WaitForNameNodesStderr: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-datanode-default-0" && .container == "wait-for-namenodes" && .file == "container.stderr.log" filteredAutomaticLogConfigJournalNode0: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-journalnode-default-0" && .container == "hdfs" + filteredAutomaticLogConfigJournalNode0Containerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-hdfs-automatic-log-journalnode-default-0" && + .container == "containerdebug" filteredAutomaticLogConfigJournalNode0Vector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-automatic-log-journalnode-default-0" && .container == "vector" filteredCustomLogConfigNameNode0: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-custom-log-namenode-default-0" && .container == "hdfs" filteredCustomLogConfigNameNode1: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-custom-log-namenode-default-1" && .container == "hdfs" filteredCustomLogConfigNameNode0Zkfc: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-custom-log-namenode-default-0" && .container == "zkfc" filteredCustomLogConfigNameNode1Zkfc: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-custom-log-namenode-default-1" && .container == "zkfc" + filteredCustomLogConfigNameNode0Containerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-hdfs-custom-log-namenode-default-0" && + .container == "containerdebug" filteredCustomLogConfigNameNode0Vector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-custom-log-namenode-default-0" && .container == "vector" + filteredCustomLogConfigNameNode1Containerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-hdfs-custom-log-namenode-default-1" && + .container == "containerdebug" filteredCustomLogConfigNameNode1Vector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-custom-log-namenode-default-1" && .container == "vector" filteredCustomLogConfigNameNode0FormatNameNodes: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-custom-log-namenode-default-0" && .container == "format-namenodes" filteredCustomLogConfigNameNode1FormatNamenodes: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-custom-log-namenode-default-1" && .container == "format-namenodes" filteredCustomLogConfigNameNode0FormatZookeeper: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-custom-log-namenode-default-0" && .container == "format-zookeeper" filteredCustomLogConfigDataNode0: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-custom-log-datanode-default-0" && .container == "hdfs" + filteredCustomLogConfigDataNode0Containerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-hdfs-custom-log-datanode-default-0" && + .container == "containerdebug" filteredCustomLogConfigDataNode0Vector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-custom-log-datanode-default-0" && .container == "vector" filteredCustomLogConfigDataNode0WaitForNameNodes: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-custom-log-datanode-default-0" && .container == "wait-for-namenodes" filteredCustomLogConfigJournalNode0: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-custom-log-journalnode-default-0" && .container == "hdfs" + filteredCustomLogConfigJournalNode0Containerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-hdfs-custom-log-journalnode-default-0" && + .container == "containerdebug" filteredCustomLogConfigJournalNode0Vector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hdfs-custom-log-journalnode-default-0" && .container == "vector" filteredInvalidEvents: type: filter - inputs: [vector] + inputs: + - vector condition: |- .timestamp == from_unix_timestamp!(0) || is_null(.level) || @@ -276,11 +372,29 @@ customConfig: is_null(.message) sinks: test: - inputs: [filtered*] + inputs: + - filtered* type: blackhole + # Shows all events received from the agents. This sink is intentional and + # not a debugging leftover: if a "filtered*" transform gathers no event, + # then the assertion only reports a count, whereas this log shows which + # pod and container names actually arrived. A failure in CI cannot be + # reproduced by enabling this afterwards, so it is kept enabled at the + # price of the log volume. + stdout: + inputs: + - vector + type: console + encoding: + codec: json + buffer: + # Avoid back pressure from a stalling stdout. The test should not + # fail because of a sink which only aids debugging. + when_full: drop_newest {% if lookup('env', 'VECTOR_AGGREGATOR') %} aggregator: - inputs: [vector] + inputs: + - vector type: vector routing: endpoints: diff --git a/tests/templates/kuttl/smoke/30-assert.yaml.j2 b/tests/templates/kuttl/smoke/30-assert.yaml.j2 index 6a5d8414..8f3d1262 100644 --- a/tests/templates/kuttl/smoke/30-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-assert.yaml.j2 @@ -319,6 +319,7 @@ spec: rm -f /stackable/log/_vector/shutdown prepare_signal_handlers + mkdir --parents "$FILE_LOG_DIRECTORY" containerdebug --output=/stackable/log/containerdebug-state.json --loop & if [[ -d /stackable/listener ]]; then export POD_ADDRESS=$(cat /stackable/listener/default-address/address) @@ -338,8 +339,12 @@ spec: env: - name: COMMON_VAR value: group-value - - name: CONTAINERDEBUG_LOG_DIRECTORY + - name: FILE_LOG_DIRECTORY value: /stackable/log/containerdebug + - name: FILE_LOG_MAX_FILES + value: "2" + - name: FILE_LOG_ROTATION_PERIOD + value: hourly - name: GROUP_VAR value: group-value - name: HADOOP_CONF_DIR @@ -482,8 +487,6 @@ spec: env: - name: COMMON_VAR value: group-value - - name: CONTAINERDEBUG_LOG_DIRECTORY - value: /stackable/log/containerdebug - name: GROUP_VAR value: group-value - name: HADOOP_CONF_DIR @@ -601,8 +604,6 @@ spec: env: - name: COMMON_VAR value: group-value - - name: CONTAINERDEBUG_LOG_DIRECTORY - value: /stackable/log/containerdebug - name: GROUP_VAR value: group-value - name: HADOOP_CONF_DIR @@ -691,8 +692,6 @@ spec: env: - name: COMMON_VAR value: group-value - - name: CONTAINERDEBUG_LOG_DIRECTORY - value: /stackable/log/containerdebug - name: GROUP_VAR value: group-value - name: HADOOP_CONF_DIR @@ -903,6 +902,7 @@ spec: rm -f /stackable/log/_vector/shutdown prepare_signal_handlers + mkdir --parents "$FILE_LOG_DIRECTORY" containerdebug --output=/stackable/log/containerdebug-state.json --loop & if [[ -d /stackable/listener ]]; then export POD_ADDRESS=$(cat /stackable/listener/default-address/address) @@ -922,8 +922,12 @@ spec: env: - name: COMMON_VAR value: group-value - - name: CONTAINERDEBUG_LOG_DIRECTORY + - name: FILE_LOG_DIRECTORY value: /stackable/log/containerdebug + - name: FILE_LOG_MAX_FILES + value: "2" + - name: FILE_LOG_ROTATION_PERIOD + value: hourly - name: GROUP_VAR value: group-value - name: HADOOP_CONF_DIR @@ -1073,8 +1077,6 @@ spec: env: - name: COMMON_VAR value: group-value - - name: CONTAINERDEBUG_LOG_DIRECTORY - value: /stackable/log/containerdebug - name: GROUP_VAR value: group-value - name: HADOOP_CONF_DIR @@ -1297,6 +1299,7 @@ spec: rm -f /stackable/log/_vector/shutdown prepare_signal_handlers + mkdir --parents "$FILE_LOG_DIRECTORY" containerdebug --output=/stackable/log/containerdebug-state.json --loop & if [[ -d /stackable/listener ]]; then export POD_ADDRESS=$(cat /stackable/listener/default-address/address) @@ -1316,8 +1319,12 @@ spec: env: - name: COMMON_VAR value: group-value - - name: CONTAINERDEBUG_LOG_DIRECTORY + - name: FILE_LOG_DIRECTORY value: /stackable/log/containerdebug + - name: FILE_LOG_MAX_FILES + value: "2" + - name: FILE_LOG_ROTATION_PERIOD + value: hourly - name: GROUP_VAR value: group-value - name: HADOOP_CONF_DIR