siegfriedweber · Nov 5, 2026
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,8 @@ All notable changes to this project will be documented in this file.
 - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#949]).
 - Coordinators now have a default affinity to the OPA Pods when OPA authorization is configured ([#924]).
 - Add support for 483 ([#956]).
+- The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to
+  the Vector aggregator ([#970]).
 
 ### Changed
 
@@ -62,6 +64,10 @@ All notable changes to this project will be documented in this file.
   when predicting the coordinator pods. Kubernetes runs a single pod for a `StatefulSet` with
   `replicas: null`, but counting it as zero left `discovery.uri` out of `config.properties`
   altogether, so no pod could find the coordinator ([#945]).
+- The `containerdebug` process writes its log file again. The operator set
+  `CONTAINERDEBUG_LOG_DIRECTORY`, which containerdebug stopped reading in its version 0.2.0 when it
+  moved to `stackable-telemetry`; the directory is now set via `FILE_LOG_DIRECTORY`. The file is
+  rotated hourly and at most two files are kept, because rotation is opt-in ([#970]).
 
 ### Removed
 
@@ -91,6 +97,7 @@ All notable changes to this project will be documented in this file.
 [#961]: https://github.com/stackabletech/trino-operator/pull/961
 [#963]: https://github.com/stackabletech/trino-operator/pull/963
 [#966]: https://github.com/stackabletech/trino-operator/pull/966
+[#970]: https://github.com/stackabletech/trino-operator/pull/970
 
 ## [26.7.0] - 2026-07-21
 

diff --git a/docs/modules/trino/pages/usage-guide/log_aggregation.adoc b/docs/modules/trino/pages/usage-guide/log_aggregation.adoc
@@ -22,4 +22,12 @@ spec:
 Currently, the logs are collected only for `server.log`.
 Logging for `http-request.log` is disabled by default.
 
+NOTE: The logs of the xref:concepts:observability/containerdebug.adoc[`containerdebug` process] are collected as well.
+They are labelled `container: containerdebug`, although no such Kubernetes container exists.
+The process runs in the background of the `trino` container.
+The label comes from `/stackable/log/containerdebug`, the directory which the operator sets with the `FILE_LOG_DIRECTORY` environment variable and from which the Vector agent derives the container name.
+Overriding `FILE_LOG_DIRECTORY` with `envOverrides` therefore changes how these logs are aggregated.
+The agent only collects files one level below `/stackable/log`, so a sibling directory such as `/stackable/log/trino` relabels the events instead of dropping them, while any deeper or shallower path stops the aggregation.
+A path outside `/stackable/log` also leaves the size-capped log volume.
+
 Further information on how to configure logging, can be found in xref:concepts:logging.adoc[].
diff --git a/rust/operator-binary/src/controller/build/command.rs b/rust/operator-binary/src/controller/build/command.rs
@@ -151,6 +151,7 @@ pub fn container_trino_args(
 {COMMON_BASH_TRAP_FUNCTIONS}
 {remove_vector_shutdown_file_command}
 prepare_signal_handlers
+mkdir --parents \"$FILE_LOG_DIRECTORY\"
 containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &
 bin/launcher run --etc-dir={RW_CONFIG_DIR_NAME} &
 wait_for_termination $!

diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs
@@ -22,6 +22,8 @@ mod tests {
         assert!(!content.is_empty());
         // The airlift source must be present (Trino's main log format) ...
         assert!(content.contains("files_airlift"));
+        assert!(content.contains("files_tracing_rs"));
+        assert!(content.contains("*.tracing-rs.json"));
         // ... while the aggregator sink must reference the injected address.
         assert!(content.contains("${VECTOR_AGGREGATOR_ADDRESS}"));
     }

diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml
@@ -99,6 +99,285 @@ tests:
                 "timestamp": t'2024-01-01T00:00:00.123456789Z'
               }
 
+              assert_eq!(expected_log_event, .)
+  - name: Test tracing-rs log
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json
+          message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":"found disk","disk.name":"overlay"},"target":"containerdebug::system_information::disk"}'
+          pod: trino-coordinator-default-0
+          source_type: file
+          timestamp: 2026-11-02T14:49:29.473487331Z
+    outputs:
+      - extract_from: extended_logs
+        expected_event_count: 1
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "trino",
+                "container": "containerdebug",
+                "file": "containerdebug.2026-11-02-14.tracing-rs.json",
+                "level": "INFO",
+                "logger": "containerdebug::system_information::disk",
+                "message": "found disk\n\ndisk.name=overlay",
+                "timestamp": t'2026-11-02T14:49:28.028922Z',
+                "namespace": "default",
+                "pod": "trino-coordinator-default-0",
+                "role": "coordinator",
+                "roleGroup": "default"
+              }
+
+              assert_eq!(expected_log_event, .)
+  - name: Test unparsable tracing-rs log
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json
+          message: '{ unparsable'
+          pod: trino-coordinator-default-0
+          source_type: file
+          timestamp: 2026-11-02T14:49:29.473487331Z
+    outputs:
+      - extract_from: extended_logs
+        expected_event_count: 1
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "trino",
+                "container": "containerdebug",
+                "errors": [
+                  .errors[0]
+                ],
+                "file": "containerdebug.2026-11-02-14.tracing-rs.json",
+                "level": "INFO",
+                "logger": "",
+                "message": "{ unparsable",
+                "timestamp": .timestamp,
+                "namespace": "default",
+                "pod": "trino-coordinator-default-0",
+                "role": "coordinator",
+                "roleGroup": "default"
+              }
+
+              assert_eq!(expected_log_event, .)
+
+              # The remainder of the message contains character offsets into
+              # the VRL source, so only the stable prefix is asserted.
+              assert!(starts_with!(.errors[0], "JSON not parsable: "))
+  - name: Test tracing-rs log which is not a JSON object
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json
+          message: 'false'
+          pod: trino-coordinator-default-0
+          source_type: file
+          timestamp: 2026-11-02T14:49:29.473487331Z
+    outputs:
+      - extract_from: extended_logs
+        expected_event_count: 1
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "trino",
+                "container": "containerdebug",
+                "errors": [
+                  "Parsed event is not a JSON object."
+                ],
+                "file": "containerdebug.2026-11-02-14.tracing-rs.json",
+                "level": "INFO",
+                "logger": "",
+                "message": "false",
+                "timestamp": .timestamp,
+                "namespace": "default",
+                "pod": "trino-coordinator-default-0",
+                "role": "coordinator",
+                "roleGroup": "default"
+              }
+
+              assert_eq!(expected_log_event, .)
+  - name: Test tracing-rs log with invalid fields
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json
+          message: '{"timestamp":"unparsable","level":"INVALID","fields":"none","target":null}'
+          pod: trino-coordinator-default-0
+          source_type: file
+          timestamp: 2026-11-02T14:49:29.473487331Z
+    outputs:
+      - extract_from: extended_logs
+        expected_event_count: 1
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "trino",
+                "container": "containerdebug",
+                "errors": [
+                  .errors[0],
+                  "Logger/target not found.",
+                  "Level \"INVALID\" unknown, using \"INFO\" instead.",
+                  "Fields are not an object."
+                ],
+                "file": "containerdebug.2026-11-02-14.tracing-rs.json",
+                "level": "INFO",
+                "logger": "",
+                "message": "{\"timestamp\":\"unparsable\",\"level\":\"INVALID\",\"fields\":\"none\",\"target\":null}",
+                "timestamp": .timestamp,
+                "namespace": "default",
+                "pod": "trino-coordinator-default-0",
+                "role": "coordinator",
+                "roleGroup": "default"
+              }
+
+              assert_eq!(expected_log_event, .)
+
+              # The remainder of the message contains character offsets into
+              # the VRL source, so only the stable prefix is asserted.
+              assert!(starts_with!(.errors[0], "Timestamp not parsable, using current time instead: "))
+  - name: Test tracing-rs log without a timestamp
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json
+          message: '{"level":"INFO","fields":{"message":"found disk"},"target":"containerdebug::system_information::disk"}'
+          pod: trino-coordinator-default-0
+          source_type: file
+          timestamp: 2026-11-02T14:49:29.473487331Z
+    outputs:
+      - extract_from: extended_logs
+        expected_event_count: 1
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "trino",
+                "container": "containerdebug",
+                "errors": [
+                  "Timestamp not found, using current time instead."
+                ],
+                "file": "containerdebug.2026-11-02-14.tracing-rs.json",
+                "level": "INFO",
+                "logger": "containerdebug::system_information::disk",
+                "message": "found disk",
+                "timestamp": .timestamp,
+                "namespace": "default",
+                "pod": "trino-coordinator-default-0",
+                "role": "coordinator",
+                "roleGroup": "default"
+              }
+
+              assert_eq!(expected_log_event, .)
+
+              # The current time is used and not the timestamp which the file
+              # source attached to the event. The latter reaches the transform
+              # as a string, whereas now() yields a timestamp.
+              assert!(is_timestamp(.timestamp))
+  - name: Test tracing-rs log without a message but with other fields
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json
+          message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}'
+          pod: trino-coordinator-default-0
+          source_type: file
+          timestamp: 2026-11-02T14:49:29.473487331Z
+    outputs:
+      - extract_from: extended_logs
+        expected_event_count: 1
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "trino",
+                "container": "containerdebug",
+                "file": "containerdebug.2026-11-02-14.tracing-rs.json",
+                "level": "INFO",
+                "logger": "containerdebug::system_information::disk",
+                "message": "disk.name=overlay",
+                "timestamp": t'2026-11-02T14:49:28.028922Z',
+                "namespace": "default",
+                "pod": "trino-coordinator-default-0",
+                "role": "coordinator",
+                "roleGroup": "default"
+              }
+
+              assert_eq!(expected_log_event, .)
+  - name: Test tracing-rs log with a message which is not a string
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json
+          message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":42,"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}'
+          pod: trino-coordinator-default-0
+          source_type: file
+          timestamp: 2026-11-02T14:49:29.473487331Z
+    outputs:
+      - extract_from: extended_logs
+        expected_event_count: 1
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "trino",
+                "container": "containerdebug",
+                "file": "containerdebug.2026-11-02-14.tracing-rs.json",
+                "level": "INFO",
+                "logger": "containerdebug::system_information::disk",
+                "message": "disk.name=overlay\nmessage=42",
+                "timestamp": t'2026-11-02T14:49:28.028922Z',
+                "namespace": "default",
+                "pod": "trino-coordinator-default-0",
+                "role": "coordinator",
+                "roleGroup": "default"
+              }
+
+              assert_eq!(expected_log_event, .)
+  - name: Test tracing-rs log without any fields
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json
+          message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{},"target":"containerdebug::system_information::disk"}'
+          pod: trino-coordinator-default-0
+          source_type: file
+          timestamp: 2026-11-02T14:49:29.473487331Z
+    outputs:
+      - extract_from: extended_logs
+        expected_event_count: 1
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "trino",
+                "container": "containerdebug",
+                "errors": [
+                  "Message not found."
+                ],
+                "file": "containerdebug.2026-11-02-14.tracing-rs.json",
+                "level": "INFO",
+                "logger": "containerdebug::system_information::disk",
+                "message": "",
+                "timestamp": t'2026-11-02T14:49:28.028922Z',
+                "namespace": "default",
+                "pod": "trino-coordinator-default-0",
+                "role": "coordinator",
+                "roleGroup": "default"
+              }
+
               assert_eq!(expected_log_event, .)
   - name: Test Vector internal logs
     inputs: