From 4062c1b52471888e98a19af76742987eb0183045 Mon Sep 17 00:00:00 2001 From: Steve Patterson Date: Mon, 27 Jul 2026 11:35:13 -0500 Subject: [PATCH 1/3] fix(transform): do not emit empty payloads from a stale aggregate aggregate.Add refreshes a.now on every successful add as well as on reset, so the configured duration behaves as an idle timeout rather than a batch age limit. Once an aggregate has sat idle for longer than that duration, the next Add returns false. aggregate_to_array treats a false Add as "flush the current batch", but in this case the batch is still empty, so aggToArray returns nil and an empty message is emitted. send_http_post then POSTs an empty HTTP body, which a receiver rejects: LogScale answers 400 "The request is not valid JSON". In the seceng LogScale pipelines this accounted for roughly 40 rejected batches per hour on the busiest source. It presents as intermittent because it depends on instance idle time; with cpu_idle enabled, Cloud Run instances routinely sit idle for over the configured minute, and the first batch after an idle gap emits one empty POST. Skips the emit when the array is empty, in both the control-flush and the mid-stream paths, and guards send_http_post against posting an empty body so that any auxiliary transform combination is covered. The underlying Add semantics are left alone deliberately: changing when a.now is refreshed would alter batching for every transform and every caller. Co-Authored-By: Claude Opus 5 --- transform/aggregate_to_array.go | 19 ++++++ transform/aggregate_to_array_stale_test.go | 70 ++++++++++++++++++++++ transform/send_http_post.go | 7 +++ 3 files changed, 96 insertions(+) create mode 100644 transform/aggregate_to_array_stale_test.go diff --git a/transform/aggregate_to_array.go b/transform/aggregate_to_array.go index bf66379..a0ae4c8 100644 --- a/transform/aggregate_to_array.go +++ b/transform/aggregate_to_array.go @@ -57,6 +57,12 @@ func (tf *aggregateToArray) Transform(ctx context.Context, msg *message.Message) for _, items := range tf.agg.GetAll() { array := aggToArray(items.Get()) + // An empty aggregate has nothing to forward. Emitting a message would + // hand an empty payload to the next transform. + if len(array) == 0 { + continue + } + outMsg := message.New() if tf.hasObjTrg { if err := outMsg.SetValue(tf.conf.Object.TargetKey, array); err != nil { @@ -82,6 +88,19 @@ func (tf *aggregateToArray) Transform(ctx context.Context, msg *message.Message) array := aggToArray(tf.agg.Get(key)) + // Add can also fail because the aggregate has been idle for longer than the + // configured duration, in which case the flush above found it empty. Emitting a + // message here would hand an empty payload to the next transform, and sinks + // cannot distinguish that from real data. + if len(array) == 0 { + tf.agg.Reset(key) + if ok := tf.agg.Add(key, msg.Data()); !ok { + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errBatchNoMoreData) + } + + return nil, nil + } + outMsg := message.New() if tf.hasObjTrg { if err := outMsg.SetValue(tf.conf.Object.TargetKey, array); err != nil { diff --git a/transform/aggregate_to_array_stale_test.go b/transform/aggregate_to_array_stale_test.go new file mode 100644 index 0000000..c56fc79 --- /dev/null +++ b/transform/aggregate_to_array_stale_test.go @@ -0,0 +1,70 @@ +package transform + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/brexhq/substation/v2/config" + "github.com/brexhq/substation/v2/message" +) + +// TestAggregateToArrayStaleBatch probes aggregate.Add's duration check. a.now +// is refreshed on every successful Add and on Reset, so once an aggregate has sat +// idle for longer than the configured duration the next Add returns false. When +// that happens on the first item of a fresh batch, aggregate_to_array flushes an +// aggregate that is still empty. +func TestAggregateToArrayStaleBatch(t *testing.T) { + ctx := context.TODO() + + // Short duration so the idle window is testable. + toArr, err := newAggregateToArray(ctx, config.Config{ + Settings: map[string]interface{}{ + "batch": map[string]interface{}{"duration": "50ms", "count": 1000, "size": 1000000}, + }, + }) + if err != nil { + t.Fatal(err) + } + + // First batch: one item, flushed by a control message. + if _, err := toArr.Transform(ctx, message.New().SetData([]byte(`{"a":1}`))); err != nil { + t.Fatal(err) + } + if _, err := toArr.Transform(ctx, message.New().AsControl()); err != nil { + t.Fatal(err) + } + + // Idle longer than the batch duration, as an idle Cloud Run instance would. + time.Sleep(120 * time.Millisecond) + + // Next item after the idle window. + out, err := toArr.Transform(ctx, message.New().SetData([]byte(`{"b":2}`))) + if err != nil { + t.Fatal(err) + } + + for _, m := range out { + if m.IsControl() { + continue + } + + d := m.Data() + t.Logf("emitted payload: %q (len=%d)", string(d), len(d)) + + if len(d) == 0 { + t.Errorf("emitted an empty payload; sinks cannot distinguish this from real data") + continue + } + + if !json.Valid(d) { + t.Errorf(" -> payload is not valid JSON: %q", string(d)) + } + } + + // Emitting nothing is the correct outcome: the stale aggregate was empty. + if len(out) != 0 { + t.Logf("emitted %d message(s) on the stale add", len(out)) + } +} diff --git a/transform/send_http_post.go b/transform/send_http_post.go index f4069e2..00eb747 100644 --- a/transform/send_http_post.go +++ b/transform/send_http_post.go @@ -187,6 +187,13 @@ func (tf *sendHTTPPost) send(ctx context.Context, key string) error { } for _, d := range data { + // Auxiliary transforms can legitimately reduce a batch to nothing, e.g. when + // every element was blank. Posting an empty body would be rejected by the + // receiver, so there is nothing to send. + if len(d) == 0 { + continue + } + start := time.Now() resp, err := tf.client.Post(ctx, url, d, headers...) if err != nil { From 8bc51ede37cd050988427ff2040f8526fc1184de Mon Sep 17 00:00:00 2001 From: Steve Patterson Date: Mon, 27 Jul 2026 11:35:13 -0500 Subject: [PATCH 2/3] fix(transform): skip blank elements when splitting and re-aggregating Splitting on a separator yields blank elements for trailing or repeated separators, and newline-delimited files normally end with a trailing newline. aggregate_from_string turned each one into a message and aggToArray joined them verbatim, producing a malformed array such as `[{"a":1},]`. In practice a following transform that sets a key, such as time_now, rescued the blank into a valid object, so the malformed array was rarely observed. Instead the blank arrived at the sink as a content-free event, e.g. {"seceng_pipeline_time":...}, injecting one junk record per blank line. Blank and whitespace-only elements are now skipped in both places. The whitespace check also covers the carriage return left behind when CRLF data is split on a newline. This is a separate defect from the empty-payload fix in the preceding commit. Co-Authored-By: Claude Opus 5 --- transform/aggregate.go | 18 +++++++- transform/aggregate_from_string.go | 8 ++++ transform/aggregate_from_string_test.go | 36 +++++++++++++++ transform/aggregate_test.go | 58 +++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 2 deletions(-) diff --git a/transform/aggregate.go b/transform/aggregate.go index fbd03d0..71d8d1e 100644 --- a/transform/aggregate.go +++ b/transform/aggregate.go @@ -19,11 +19,25 @@ func (c *aggregateArrayConfig) Decode(in interface{}) error { } func aggToArray(data [][]byte) []byte { - if len(data) == 0 { + // Blank elements are excluded because joining them produces a malformed + // array, e.g. `[{"a":1},]`, which downstream consumers reject. Splitting on a + // separator yields blank elements for trailing or repeated separators, so + // callers can legitimately pass them. + items := make([][]byte, 0, len(data)) + + for _, d := range data { + if len(bytes.TrimSpace(d)) == 0 { + continue + } + + items = append(items, d) + } + + if len(items) == 0 { return nil } - return slices.Concat([]byte("["), bytes.Join(data, []byte(",")), []byte("]")) + return slices.Concat([]byte("["), bytes.Join(items, []byte(",")), []byte("]")) } type aggregateStrConfig struct { diff --git a/transform/aggregate_from_string.go b/transform/aggregate_from_string.go index b6a7164..05d24cf 100644 --- a/transform/aggregate_from_string.go +++ b/transform/aggregate_from_string.go @@ -1,6 +1,7 @@ package transform import ( + "bytes" "context" "encoding/json" "fmt" @@ -46,6 +47,13 @@ func (tf *aggregateFromString) Transform(ctx context.Context, msg *message.Messa deagg := aggFromStr(msg.Data(), tf.separator) for _, b := range deagg { + // Trailing or repeated separators produce blank elements that carry no + // data. They are skipped so that they do not reach downstream sinks, where + // re-aggregating them would build a malformed array. + if len(bytes.TrimSpace(b)) == 0 { + continue + } + msg := message.New().SetData(b).SetMetadata(msg.Metadata()) output = append(output, msg) } diff --git a/transform/aggregate_from_string_test.go b/transform/aggregate_from_string_test.go index f536aba..355665c 100644 --- a/transform/aggregate_from_string_test.go +++ b/transform/aggregate_from_string_test.go @@ -33,6 +33,42 @@ var aggregateFromStringTests = []struct { `{"e":"f"}`, }, }, + { + // Newline-delimited files normally end with a trailing separator. The blank + // element it produces must not become a message, or re-aggregating it + // downstream builds a malformed array. + "trailing separator", + config.Config{ + Settings: map[string]interface{}{ + "separator": `\n`, + }, + }, + []string{ + `{"a":"b"}\n{"c":"d"}\n`, + }, + []string{ + `{"a":"b"}`, + `{"c":"d"}`, + }, + }, + { + // Blank lines within a file produce blank elements too. Whitespace-only + // elements, such as the carriage return left when CRLF data is split on a + // newline, are covered by TestAggToArray. + "repeated separators", + config.Config{ + Settings: map[string]interface{}{ + "separator": `\n`, + }, + }, + []string{ + `{"a":"b"}\n\n{"c":"d"}\n`, + }, + []string{ + `{"a":"b"}`, + `{"c":"d"}`, + }, + }, } func TestAggregateFromString(t *testing.T) { diff --git a/transform/aggregate_test.go b/transform/aggregate_test.go index d2f7b04..2563a1e 100644 --- a/transform/aggregate_test.go +++ b/transform/aggregate_test.go @@ -1,9 +1,67 @@ package transform import ( + "encoding/json" "testing" ) +var aggToArrayTests = []struct { + name string + data []string + expected string +}{ + { + "objects", + []string{`{"a":1}`, `{"b":2}`}, + `[{"a":1},{"b":2}]`, + }, + { + // A trailing separator in the source yields a blank element. Joining it + // verbatim produces `[{"a":1},{"b":2},]`, which is not valid JSON. + "trailing blank element", + []string{`{"a":1}`, `{"b":2}`, ``}, + `[{"a":1},{"b":2}]`, + }, + { + "interior blank element", + []string{`{"a":1}`, ``, `{"b":2}`}, + `[{"a":1},{"b":2}]`, + }, + { + // Splitting CRLF data on a newline leaves a carriage return behind. + "whitespace-only element", + []string{`{"a":1}`, "\r"}, + `[{"a":1}]`, + }, + { + "only blank elements", + []string{``, ``}, + ``, + }, +} + +func TestAggToArray(t *testing.T) { + for _, test := range aggToArrayTests { + t.Run(test.name, func(t *testing.T) { + var data [][]byte + for _, d := range test.data { + data = append(data, []byte(d)) + } + + result := aggToArray(data) + + if string(result) != test.expected { + t.Errorf("expected %q, got %q", test.expected, string(result)) + } + + // Any non-empty result must be valid JSON; that is the point of the test. + if len(result) > 0 && !json.Valid(result) { + t.Errorf("result is not valid JSON: %q", string(result)) + } + }) + } +} + func TestAggregateArrayConfigDecode(t *testing.T) { config := &aggregateArrayConfig{} err := config.Decode(map[string]interface{}{ From 1894d4ae6c0fea7a08435ea29555988e3890f323 Mon Sep 17 00:00:00 2001 From: Steve Patterson Date: Mon, 27 Jul 2026 11:43:35 -0500 Subject: [PATCH 3/3] refactor(transform): extract newArrayMessage from aggregateToArray Guarding against the empty aggregate added a nested branch inside the control flush, which pushed the block past the nestif threshold. Extracting the message construction removes the nesting and drops the duplication between the control and mid-stream paths, which built the same message two different ways. No behaviour change; TestAggregateToArrayStaleBatch still fails against the pre-fix aggregate_to_array.go and passes here. Co-Authored-By: Claude Opus 5 --- transform/aggregate_to_array.go | 66 +++++++++++++++++---------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/transform/aggregate_to_array.go b/transform/aggregate_to_array.go index a0ae4c8..39da60c 100644 --- a/transform/aggregate_to_array.go +++ b/transform/aggregate_to_array.go @@ -55,21 +55,13 @@ func (tf *aggregateToArray) Transform(ctx context.Context, msg *message.Message) var output []*message.Message for _, items := range tf.agg.GetAll() { - array := aggToArray(items.Get()) - - // An empty aggregate has nothing to forward. Emitting a message would - // hand an empty payload to the next transform. - if len(array) == 0 { - continue + outMsg, err := tf.newArrayMessage(items.Get()) + if err != nil { + return nil, err } - outMsg := message.New() - if tf.hasObjTrg { - if err := outMsg.SetValue(tf.conf.Object.TargetKey, array); err != nil { - return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) - } - } else { - outMsg.SetData(array) + if outMsg == nil { + continue } output = append(output, outMsg) @@ -86,18 +78,34 @@ func (tf *aggregateToArray) Transform(ctx context.Context, msg *message.Message) return nil, nil } - array := aggToArray(tf.agg.Get(key)) - // Add can also fail because the aggregate has been idle for longer than the - // configured duration, in which case the flush above found it empty. Emitting a - // message here would hand an empty payload to the next transform, and sinks - // cannot distinguish that from real data. - if len(array) == 0 { - tf.agg.Reset(key) - if ok := tf.agg.Add(key, msg.Data()); !ok { - return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errBatchNoMoreData) - } + // configured duration, in which case the aggregate is still empty and + // newArrayMessage returns nil rather than an empty payload. + outMsg, err := tf.newArrayMessage(tf.agg.Get(key)) + if err != nil { + return nil, err + } + + // If data cannot be added after reset, then the batch is misconfgured. + tf.agg.Reset(key) + if ok := tf.agg.Add(key, msg.Data()); !ok { + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errBatchNoMoreData) + } + if outMsg == nil { + return nil, nil + } + + return []*message.Message{outMsg}, nil +} + +// newArrayMessage builds a message holding the aggregated array, or nil when the +// aggregate is empty. An empty payload is not forwarded because sinks cannot +// distinguish it from real data; send_http_post, for example, would POST an empty +// body. +func (tf *aggregateToArray) newArrayMessage(items [][]byte) (*message.Message, error) { + array := aggToArray(items) + if len(array) == 0 { return nil, nil } @@ -106,17 +114,13 @@ func (tf *aggregateToArray) Transform(ctx context.Context, msg *message.Message) if err := outMsg.SetValue(tf.conf.Object.TargetKey, array); err != nil { return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } - } else { - outMsg.SetData(array) - } - // If data cannot be added after reset, then the batch is misconfgured. - tf.agg.Reset(key) - if ok := tf.agg.Add(key, msg.Data()); !ok { - return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errBatchNoMoreData) + return outMsg, nil } - return []*message.Message{outMsg}, nil + outMsg.SetData(array) + + return outMsg, nil } func (tf *aggregateToArray) String() string {