diff --git a/transform/aggregate.go b/transform/aggregate.go index fbd03d0..71d8d1e 100644 --- a/transform/aggregate.go +++ b/transform/aggregate.go @@ -19,11 +19,25 @@ func (c *aggregateArrayConfig) Decode(in interface{}) error { } func aggToArray(data [][]byte) []byte { - if len(data) == 0 { + // Blank elements are excluded because joining them produces a malformed + // array, e.g. `[{"a":1},]`, which downstream consumers reject. Splitting on a + // separator yields blank elements for trailing or repeated separators, so + // callers can legitimately pass them. + items := make([][]byte, 0, len(data)) + + for _, d := range data { + if len(bytes.TrimSpace(d)) == 0 { + continue + } + + items = append(items, d) + } + + if len(items) == 0 { return nil } - return slices.Concat([]byte("["), bytes.Join(data, []byte(",")), []byte("]")) + return slices.Concat([]byte("["), bytes.Join(items, []byte(",")), []byte("]")) } type aggregateStrConfig struct { diff --git a/transform/aggregate_from_string.go b/transform/aggregate_from_string.go index b6a7164..05d24cf 100644 --- a/transform/aggregate_from_string.go +++ b/transform/aggregate_from_string.go @@ -1,6 +1,7 @@ package transform import ( + "bytes" "context" "encoding/json" "fmt" @@ -46,6 +47,13 @@ func (tf *aggregateFromString) Transform(ctx context.Context, msg *message.Messa deagg := aggFromStr(msg.Data(), tf.separator) for _, b := range deagg { + // Trailing or repeated separators produce blank elements that carry no + // data. They are skipped so that they do not reach downstream sinks, where + // re-aggregating them would build a malformed array. + if len(bytes.TrimSpace(b)) == 0 { + continue + } + msg := message.New().SetData(b).SetMetadata(msg.Metadata()) output = append(output, msg) } diff --git a/transform/aggregate_from_string_test.go b/transform/aggregate_from_string_test.go index f536aba..355665c 100644 --- a/transform/aggregate_from_string_test.go +++ b/transform/aggregate_from_string_test.go @@ -33,6 +33,42 @@ var aggregateFromStringTests = []struct { `{"e":"f"}`, }, }, + { + // Newline-delimited files normally end with a trailing separator. The blank + // element it produces must not become a message, or re-aggregating it + // downstream builds a malformed array. + "trailing separator", + config.Config{ + Settings: map[string]interface{}{ + "separator": `\n`, + }, + }, + []string{ + `{"a":"b"}\n{"c":"d"}\n`, + }, + []string{ + `{"a":"b"}`, + `{"c":"d"}`, + }, + }, + { + // Blank lines within a file produce blank elements too. Whitespace-only + // elements, such as the carriage return left when CRLF data is split on a + // newline, are covered by TestAggToArray. + "repeated separators", + config.Config{ + Settings: map[string]interface{}{ + "separator": `\n`, + }, + }, + []string{ + `{"a":"b"}\n\n{"c":"d"}\n`, + }, + []string{ + `{"a":"b"}`, + `{"c":"d"}`, + }, + }, } func TestAggregateFromString(t *testing.T) { diff --git a/transform/aggregate_test.go b/transform/aggregate_test.go index d2f7b04..2563a1e 100644 --- a/transform/aggregate_test.go +++ b/transform/aggregate_test.go @@ -1,9 +1,67 @@ package transform import ( + "encoding/json" "testing" ) +var aggToArrayTests = []struct { + name string + data []string + expected string +}{ + { + "objects", + []string{`{"a":1}`, `{"b":2}`}, + `[{"a":1},{"b":2}]`, + }, + { + // A trailing separator in the source yields a blank element. Joining it + // verbatim produces `[{"a":1},{"b":2},]`, which is not valid JSON. + "trailing blank element", + []string{`{"a":1}`, `{"b":2}`, ``}, + `[{"a":1},{"b":2}]`, + }, + { + "interior blank element", + []string{`{"a":1}`, ``, `{"b":2}`}, + `[{"a":1},{"b":2}]`, + }, + { + // Splitting CRLF data on a newline leaves a carriage return behind. + "whitespace-only element", + []string{`{"a":1}`, "\r"}, + `[{"a":1}]`, + }, + { + "only blank elements", + []string{``, ``}, + ``, + }, +} + +func TestAggToArray(t *testing.T) { + for _, test := range aggToArrayTests { + t.Run(test.name, func(t *testing.T) { + var data [][]byte + for _, d := range test.data { + data = append(data, []byte(d)) + } + + result := aggToArray(data) + + if string(result) != test.expected { + t.Errorf("expected %q, got %q", test.expected, string(result)) + } + + // Any non-empty result must be valid JSON; that is the point of the test. + if len(result) > 0 && !json.Valid(result) { + t.Errorf("result is not valid JSON: %q", string(result)) + } + }) + } +} + func TestAggregateArrayConfigDecode(t *testing.T) { config := &aggregateArrayConfig{} err := config.Decode(map[string]interface{}{ diff --git a/transform/aggregate_to_array.go b/transform/aggregate_to_array.go index bf66379..39da60c 100644 --- a/transform/aggregate_to_array.go +++ b/transform/aggregate_to_array.go @@ -55,15 +55,13 @@ func (tf *aggregateToArray) Transform(ctx context.Context, msg *message.Message) var output []*message.Message for _, items := range tf.agg.GetAll() { - array := aggToArray(items.Get()) - - outMsg := message.New() - if tf.hasObjTrg { - if err := outMsg.SetValue(tf.conf.Object.TargetKey, array); err != nil { - return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) - } - } else { - outMsg.SetData(array) + outMsg, err := tf.newArrayMessage(items.Get()) + if err != nil { + return nil, err + } + + if outMsg == nil { + continue } output = append(output, outMsg) @@ -80,15 +78,12 @@ func (tf *aggregateToArray) Transform(ctx context.Context, msg *message.Message) return nil, nil } - array := aggToArray(tf.agg.Get(key)) - - outMsg := message.New() - if tf.hasObjTrg { - if err := outMsg.SetValue(tf.conf.Object.TargetKey, array); err != nil { - return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) - } - } else { - outMsg.SetData(array) + // Add can also fail because the aggregate has been idle for longer than the + // configured duration, in which case the aggregate is still empty and + // newArrayMessage returns nil rather than an empty payload. + outMsg, err := tf.newArrayMessage(tf.agg.Get(key)) + if err != nil { + return nil, err } // If data cannot be added after reset, then the batch is misconfgured. @@ -97,9 +92,37 @@ func (tf *aggregateToArray) Transform(ctx context.Context, msg *message.Message) return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errBatchNoMoreData) } + if outMsg == nil { + return nil, nil + } + return []*message.Message{outMsg}, nil } +// newArrayMessage builds a message holding the aggregated array, or nil when the +// aggregate is empty. An empty payload is not forwarded because sinks cannot +// distinguish it from real data; send_http_post, for example, would POST an empty +// body. +func (tf *aggregateToArray) newArrayMessage(items [][]byte) (*message.Message, error) { + array := aggToArray(items) + if len(array) == 0 { + return nil, nil + } + + outMsg := message.New() + if tf.hasObjTrg { + if err := outMsg.SetValue(tf.conf.Object.TargetKey, array); err != nil { + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) + } + + return outMsg, nil + } + + outMsg.SetData(array) + + return outMsg, nil +} + func (tf *aggregateToArray) String() string { b, _ := json.Marshal(tf.conf) return string(b) diff --git a/transform/aggregate_to_array_stale_test.go b/transform/aggregate_to_array_stale_test.go new file mode 100644 index 0000000..c56fc79 --- /dev/null +++ b/transform/aggregate_to_array_stale_test.go @@ -0,0 +1,70 @@ +package transform + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/brexhq/substation/v2/config" + "github.com/brexhq/substation/v2/message" +) + +// TestAggregateToArrayStaleBatch probes aggregate.Add's duration check. a.now +// is refreshed on every successful Add and on Reset, so once an aggregate has sat +// idle for longer than the configured duration the next Add returns false. When +// that happens on the first item of a fresh batch, aggregate_to_array flushes an +// aggregate that is still empty. +func TestAggregateToArrayStaleBatch(t *testing.T) { + ctx := context.TODO() + + // Short duration so the idle window is testable. + toArr, err := newAggregateToArray(ctx, config.Config{ + Settings: map[string]interface{}{ + "batch": map[string]interface{}{"duration": "50ms", "count": 1000, "size": 1000000}, + }, + }) + if err != nil { + t.Fatal(err) + } + + // First batch: one item, flushed by a control message. + if _, err := toArr.Transform(ctx, message.New().SetData([]byte(`{"a":1}`))); err != nil { + t.Fatal(err) + } + if _, err := toArr.Transform(ctx, message.New().AsControl()); err != nil { + t.Fatal(err) + } + + // Idle longer than the batch duration, as an idle Cloud Run instance would. + time.Sleep(120 * time.Millisecond) + + // Next item after the idle window. + out, err := toArr.Transform(ctx, message.New().SetData([]byte(`{"b":2}`))) + if err != nil { + t.Fatal(err) + } + + for _, m := range out { + if m.IsControl() { + continue + } + + d := m.Data() + t.Logf("emitted payload: %q (len=%d)", string(d), len(d)) + + if len(d) == 0 { + t.Errorf("emitted an empty payload; sinks cannot distinguish this from real data") + continue + } + + if !json.Valid(d) { + t.Errorf(" -> payload is not valid JSON: %q", string(d)) + } + } + + // Emitting nothing is the correct outcome: the stale aggregate was empty. + if len(out) != 0 { + t.Logf("emitted %d message(s) on the stale add", len(out)) + } +} diff --git a/transform/send_http_post.go b/transform/send_http_post.go index f4069e2..00eb747 100644 --- a/transform/send_http_post.go +++ b/transform/send_http_post.go @@ -187,6 +187,13 @@ func (tf *sendHTTPPost) send(ctx context.Context, key string) error { } for _, d := range data { + // Auxiliary transforms can legitimately reduce a batch to nothing, e.g. when + // every element was blank. Posting an empty body would be rejected by the + // receiver, so there is nothing to send. + if len(d) == 0 { + continue + } + start := time.Now() resp, err := tf.client.Post(ctx, url, d, headers...) if err != nil {