Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions transform/aggregate.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,25 @@ func (c *aggregateArrayConfig) Decode(in interface{}) error {
}

func aggToArray(data [][]byte) []byte {
if len(data) == 0 {
// Blank elements are excluded because joining them produces a malformed
// array, e.g. `[{"a":1},]`, which downstream consumers reject. Splitting on a
// separator yields blank elements for trailing or repeated separators, so
// callers can legitimately pass them.
items := make([][]byte, 0, len(data))

for _, d := range data {
if len(bytes.TrimSpace(d)) == 0 {
continue
}

items = append(items, d)
}

if len(items) == 0 {
return nil
}

return slices.Concat([]byte("["), bytes.Join(data, []byte(",")), []byte("]"))
return slices.Concat([]byte("["), bytes.Join(items, []byte(",")), []byte("]"))
}

type aggregateStrConfig struct {
Expand Down
8 changes: 8 additions & 0 deletions transform/aggregate_from_string.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package transform

import (
"bytes"
"context"
"encoding/json"
"fmt"
Expand Down Expand Up @@ -46,6 +47,13 @@ func (tf *aggregateFromString) Transform(ctx context.Context, msg *message.Messa
deagg := aggFromStr(msg.Data(), tf.separator)

for _, b := range deagg {
// Trailing or repeated separators produce blank elements that carry no
// data. They are skipped so that they do not reach downstream sinks, where
// re-aggregating them would build a malformed array.
if len(bytes.TrimSpace(b)) == 0 {
continue
}

msg := message.New().SetData(b).SetMetadata(msg.Metadata())
output = append(output, msg)
}
Expand Down
36 changes: 36 additions & 0 deletions transform/aggregate_from_string_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,42 @@ var aggregateFromStringTests = []struct {
`{"e":"f"}`,
},
},
{
// Newline-delimited files normally end with a trailing separator. The blank
// element it produces must not become a message, or re-aggregating it
// downstream builds a malformed array.
"trailing separator",
config.Config{
Settings: map[string]interface{}{
"separator": `\n`,
},
},
[]string{
`{"a":"b"}\n{"c":"d"}\n`,
},
[]string{
`{"a":"b"}`,
`{"c":"d"}`,
},
},
{
// Blank lines within a file produce blank elements too. Whitespace-only
// elements, such as the carriage return left when CRLF data is split on a
// newline, are covered by TestAggToArray.
"repeated separators",
config.Config{
Settings: map[string]interface{}{
"separator": `\n`,
},
},
[]string{
`{"a":"b"}\n\n{"c":"d"}\n`,
},
[]string{
`{"a":"b"}`,
`{"c":"d"}`,
},
},
}

func TestAggregateFromString(t *testing.T) {
Expand Down
58 changes: 58 additions & 0 deletions transform/aggregate_test.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,67 @@
package transform

import (
"encoding/json"
"testing"
)

var aggToArrayTests = []struct {
name string
data []string
expected string
}{
{
"objects",
[]string{`{"a":1}`, `{"b":2}`},
`[{"a":1},{"b":2}]`,
},
{
// A trailing separator in the source yields a blank element. Joining it
// verbatim produces `[{"a":1},{"b":2},]`, which is not valid JSON.
"trailing blank element",
[]string{`{"a":1}`, `{"b":2}`, ``},
`[{"a":1},{"b":2}]`,
},
{
"interior blank element",
[]string{`{"a":1}`, ``, `{"b":2}`},
`[{"a":1},{"b":2}]`,
},
{
// Splitting CRLF data on a newline leaves a carriage return behind.
"whitespace-only element",
[]string{`{"a":1}`, "\r"},
`[{"a":1}]`,
},
{
"only blank elements",
[]string{``, ``},
``,
},
}

func TestAggToArray(t *testing.T) {
for _, test := range aggToArrayTests {
t.Run(test.name, func(t *testing.T) {
var data [][]byte
for _, d := range test.data {
data = append(data, []byte(d))
}

result := aggToArray(data)

if string(result) != test.expected {
t.Errorf("expected %q, got %q", test.expected, string(result))
}

// Any non-empty result must be valid JSON; that is the point of the test.
if len(result) > 0 && !json.Valid(result) {
t.Errorf("result is not valid JSON: %q", string(result))
}
})
}
}

func TestAggregateArrayConfigDecode(t *testing.T) {
config := &aggregateArrayConfig{}
err := config.Decode(map[string]interface{}{
Expand Down
59 changes: 41 additions & 18 deletions transform/aggregate_to_array.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,15 +55,13 @@ func (tf *aggregateToArray) Transform(ctx context.Context, msg *message.Message)
var output []*message.Message

for _, items := range tf.agg.GetAll() {
array := aggToArray(items.Get())

outMsg := message.New()
if tf.hasObjTrg {
if err := outMsg.SetValue(tf.conf.Object.TargetKey, array); err != nil {
return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err)
}
} else {
outMsg.SetData(array)
outMsg, err := tf.newArrayMessage(items.Get())
if err != nil {
return nil, err
}

if outMsg == nil {
continue
}

output = append(output, outMsg)
Expand All @@ -80,15 +78,12 @@ func (tf *aggregateToArray) Transform(ctx context.Context, msg *message.Message)
return nil, nil
}

array := aggToArray(tf.agg.Get(key))

outMsg := message.New()
if tf.hasObjTrg {
if err := outMsg.SetValue(tf.conf.Object.TargetKey, array); err != nil {
return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err)
}
} else {
outMsg.SetData(array)
// Add can also fail because the aggregate has been idle for longer than the
// configured duration, in which case the aggregate is still empty and
// newArrayMessage returns nil rather than an empty payload.
outMsg, err := tf.newArrayMessage(tf.agg.Get(key))
if err != nil {
return nil, err
}

// If data cannot be added after reset, then the batch is misconfgured.
Expand All @@ -97,9 +92,37 @@ func (tf *aggregateToArray) Transform(ctx context.Context, msg *message.Message)
return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errBatchNoMoreData)
}

if outMsg == nil {
return nil, nil
}

return []*message.Message{outMsg}, nil
}

// newArrayMessage builds a message holding the aggregated array, or nil when the
// aggregate is empty. An empty payload is not forwarded because sinks cannot
// distinguish it from real data; send_http_post, for example, would POST an empty
// body.
func (tf *aggregateToArray) newArrayMessage(items [][]byte) (*message.Message, error) {
array := aggToArray(items)
if len(array) == 0 {
return nil, nil
}

outMsg := message.New()
if tf.hasObjTrg {
if err := outMsg.SetValue(tf.conf.Object.TargetKey, array); err != nil {
return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err)
}

return outMsg, nil
}

outMsg.SetData(array)

return outMsg, nil
}

func (tf *aggregateToArray) String() string {
b, _ := json.Marshal(tf.conf)
return string(b)
Expand Down
70 changes: 70 additions & 0 deletions transform/aggregate_to_array_stale_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package transform

import (
"context"
"encoding/json"
"testing"
"time"

"github.com/brexhq/substation/v2/config"
"github.com/brexhq/substation/v2/message"
)

// TestAggregateToArrayStaleBatch probes aggregate.Add's duration check. a.now
// is refreshed on every successful Add and on Reset, so once an aggregate has sat
// idle for longer than the configured duration the next Add returns false. When
// that happens on the first item of a fresh batch, aggregate_to_array flushes an
// aggregate that is still empty.
func TestAggregateToArrayStaleBatch(t *testing.T) {
ctx := context.TODO()

// Short duration so the idle window is testable.
toArr, err := newAggregateToArray(ctx, config.Config{
Settings: map[string]interface{}{
"batch": map[string]interface{}{"duration": "50ms", "count": 1000, "size": 1000000},
},
})
if err != nil {
t.Fatal(err)
}

// First batch: one item, flushed by a control message.
if _, err := toArr.Transform(ctx, message.New().SetData([]byte(`{"a":1}`))); err != nil {
t.Fatal(err)
}
if _, err := toArr.Transform(ctx, message.New().AsControl()); err != nil {
t.Fatal(err)
}

// Idle longer than the batch duration, as an idle Cloud Run instance would.
time.Sleep(120 * time.Millisecond)

// Next item after the idle window.
out, err := toArr.Transform(ctx, message.New().SetData([]byte(`{"b":2}`)))
if err != nil {
t.Fatal(err)
}

for _, m := range out {
if m.IsControl() {
continue
}

d := m.Data()
t.Logf("emitted payload: %q (len=%d)", string(d), len(d))

if len(d) == 0 {
t.Errorf("emitted an empty payload; sinks cannot distinguish this from real data")
continue
}

if !json.Valid(d) {
t.Errorf(" -> payload is not valid JSON: %q", string(d))
}
}

// Emitting nothing is the correct outcome: the stale aggregate was empty.
if len(out) != 0 {
t.Logf("emitted %d message(s) on the stale add", len(out))
}
}
7 changes: 7 additions & 0 deletions transform/send_http_post.go
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,13 @@ func (tf *sendHTTPPost) send(ctx context.Context, key string) error {
}

for _, d := range data {
// Auxiliary transforms can legitimately reduce a batch to nothing, e.g. when
// every element was blank. Posting an empty body would be rejected by the
// receiver, so there is nothing to send.
if len(d) == 0 {
continue
}

start := time.Now()
resp, err := tf.client.Post(ctx, url, d, headers...)
if err != nil {
Expand Down
Loading