diff --git a/CHANGELOG.md b/CHANGELOG.md index 235d2b96..a020747c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ All notable changes to this project will be documented in this file. - Support floating tags for product images via the new `spec.image.stackableVersionPolicy` field ([#994]). - Add support for 2.12.0 ([#997]). +- The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to + the Vector aggregator ([#1020]). - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#999]). - Nodes now have a default affinity to the OPA Pods when OPA authorization is configured ([#1002]). @@ -56,6 +58,10 @@ All notable changes to this project will be documented in this file. See [our internal issue](https://github.com/stackabletech/hdfs-operator/issues/626) and [the fix](https://github.com/kube-rs/kube/pull/2042) for details ([#975]). - The operator now watches all resources that it creates and early-exits the reconcile action when the cluster is marked for deletion ([#985]). +- The `containerdebug` process writes its log file again. The operator set + `CONTAINERDEBUG_LOG_DIRECTORY`, which containerdebug stopped reading in its version 0.2.0 when it + moved to `stackable-telemetry`; the directory is now set via `FILE_LOG_DIRECTORY`. The file is + rotated hourly and at most two files are kept, because rotation is opt-in ([#1020]). ### Removed @@ -82,6 +88,7 @@ All notable changes to this project will be documented in this file. [#1013]: https://github.com/stackabletech/nifi-operator/pull/1013 [#1015]: https://github.com/stackabletech/nifi-operator/pull/1015 [#1017]: https://github.com/stackabletech/nifi-operator/pull/1017 +[#1020]: https://github.com/stackabletech/nifi-operator/pull/1020 ## [26.7.0] - 2026-07-21 diff --git a/docs/modules/nifi/pages/usage_guide/log-aggregation.adoc b/docs/modules/nifi/pages/usage_guide/log-aggregation.adoc index bffb7001..a8669a9e 100644 --- a/docs/modules/nifi/pages/usage_guide/log-aggregation.adoc +++ b/docs/modules/nifi/pages/usage_guide/log-aggregation.adoc @@ -14,4 +14,12 @@ spec: enableVectorAgent: true ---- +NOTE: The logs of the xref:concepts:observability/containerdebug.adoc[`containerdebug` process] are collected as well. +They are labelled `container: containerdebug`, although no such Kubernetes container exists. +The process runs in the background of the `nifi` container. +The label comes from `/stackable/log/containerdebug`, the directory which the operator sets with the `FILE_LOG_DIRECTORY` environment variable and from which the Vector agent derives the container name. +Overriding `FILE_LOG_DIRECTORY` with `envOverrides` therefore changes how these logs are aggregated. +The agent only collects files one level below `/stackable/log`, so a sibling directory such as `/stackable/log/nifi` relabels the events instead of dropping them, while any deeper or shallower path stops the aggregation. +A path outside `/stackable/log` also leaves the log volume. + Further information on how to configure logging, can be found in xref:concepts:logging.adoc[]. diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs index 931984cd..e1bb15e8 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs @@ -79,5 +79,7 @@ mod tests { assert!(!content.is_empty()); // NiFi logs via logback's `XMLLayout` (log4j XML), so the `files_log4j` source assert!(content.contains("files_log4j")); + assert!(content.contains("files_tracing_rs")); + assert!(content.contains("*.tracing-rs.json")); } } diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml index 3ac29fd1..872ccf41 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml @@ -101,6 +101,285 @@ tests: "timestamp": t'2024-01-01T00:00:00Z' } + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":"found disk","disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: nifi-node-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "nifi", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "found disk\n\ndisk.name=overlay", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "nifi-node-default-0", + "role": "node", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Test unparsable tracing-rs log + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{ unparsable' + pod: nifi-node-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "nifi", + "container": "containerdebug", + "errors": [ + .errors[0] + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "{ unparsable", + "timestamp": .timestamp, + "namespace": "default", + "pod": "nifi-node-default-0", + "role": "node", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The remainder of the message contains character offsets into + # the VRL source, so only the stable prefix is asserted. + assert!(starts_with!(.errors[0], "JSON not parsable: ")) + - name: Test tracing-rs log which is not a JSON object + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: 'false' + pod: nifi-node-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "nifi", + "container": "containerdebug", + "errors": [ + "Parsed event is not a JSON object." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "false", + "timestamp": .timestamp, + "namespace": "default", + "pod": "nifi-node-default-0", + "role": "node", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log with invalid fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"unparsable","level":"INVALID","fields":"none","target":null}' + pod: nifi-node-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "nifi", + "container": "containerdebug", + "errors": [ + .errors[0], + "Logger/target not found.", + "Level \"INVALID\" unknown, using \"INFO\" instead.", + "Fields are not an object." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "{\"timestamp\":\"unparsable\",\"level\":\"INVALID\",\"fields\":\"none\",\"target\":null}", + "timestamp": .timestamp, + "namespace": "default", + "pod": "nifi-node-default-0", + "role": "node", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The remainder of the message contains character offsets into + # the VRL source, so only the stable prefix is asserted. + assert!(starts_with!(.errors[0], "Timestamp not parsable, using current time instead: ")) + - name: Test tracing-rs log without a timestamp + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"level":"INFO","fields":{"message":"found disk"},"target":"containerdebug::system_information::disk"}' + pod: nifi-node-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "nifi", + "container": "containerdebug", + "errors": [ + "Timestamp not found, using current time instead." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "found disk", + "timestamp": .timestamp, + "namespace": "default", + "pod": "nifi-node-default-0", + "role": "node", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The current time is used and not the timestamp which the file + # source attached to the event. The latter reaches the transform + # as a string, whereas now() yields a timestamp. + assert!(is_timestamp(.timestamp)) + - name: Test tracing-rs log without a message but with other fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: nifi-node-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "nifi", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "disk.name=overlay", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "nifi-node-default-0", + "role": "node", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log with a message which is not a string + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":42,"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: nifi-node-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "nifi", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "disk.name=overlay\nmessage=42", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "nifi-node-default-0", + "role": "node", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log without any fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{},"target":"containerdebug::system_information::disk"}' + pod: nifi-node-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "nifi", + "container": "containerdebug", + "errors": [ + "Message not found." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "nifi-node-default-0", + "role": "node", + "roleGroup": "default" + } + assert_eq!(expected_log_event, .) - name: Test Vector internal logs inputs: diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml index d42c109f..3cd35ac8 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml @@ -31,6 +31,12 @@ sources: condition_pattern: ^- .pod == "test-nifi-node-automatic-log-config-0" && .container == "nifi" filteredAutomaticLogConfigNodeGitSync: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-nifi-node-automatic-log-config-0" && .container == "git-sync-0" filteredAutomaticLogConfigNodeGitSyncInit: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-nifi-node-automatic-log-config-0" && .container == "git-sync-0-init" + filteredAutomaticLogConfigNodeContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-nifi-node-automatic-log-config-0" && + .container == "containerdebug" filteredAutomaticLogConfigNodeVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-nifi-node-automatic-log-config-0" && .container == "vector" filteredCustomLogConfigNodeNifi: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-nifi-node-custom-log-config-0" && .container == "nifi" + filteredCustomLogConfigNodeContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-nifi-node-custom-log-config-0" && + .container == "containerdebug" filteredCustomLogConfigNodeVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-nifi-node-custom-log-config-0" && .container == "vector" filteredInvalidEvents: type: filter - inputs: [vector] + inputs: + - vector condition: |- .timestamp == from_unix_timestamp!(0) || is_null(.level) || @@ -70,11 +92,29 @@ customConfig: is_null(.message) sinks: test: - inputs: [filtered*] + inputs: + - filtered* type: blackhole + # Shows all events received from the agents. This sink is intentional and + # not a debugging leftover: if a "filtered*" transform gathers no event, + # then the assertion only reports a count, whereas this log shows which + # pod and container names actually arrived. A failure in CI cannot be + # reproduced by enabling this afterwards, so it is kept enabled at the + # price of the log volume. + stdout: + inputs: + - vector + type: console + encoding: + codec: json + buffer: + # Avoid back pressure from a stalling stdout. The test should not + # fail because of a sink which only aids debugging. + when_full: drop_newest {% if lookup('env', 'VECTOR_AGGREGATOR') %} aggregator: - inputs: [vector] + inputs: + - vector type: vector routing: endpoints: