FIX: Strip UTF-8 BOM in csv converter to prevent parse errors

internal/pkg/pipeline/task/converter/README.md

@@ -10,10 +10,11 @@ The converter task transforms data from one format to another, enabling interope
 
 The converter task transforms data between different formats. It receives records from its input channel, converts the data from the source format to the target format using the specified options, and sends the converted records to its output channel.
 
-- If skip_first is True and no columns are provided, then the column names in the output will be the values in the first row of the CSV file.
+- If skip_first is True and no columns are provided, then the column names in the output will be the values in the first row of the CSV file, normalized: non-alphanumeric characters are replaced by underscores, leading/trailing underscores are trimmed, and the result is lowercased.
 - If skip_first is True and columns are provided, then the column names in the output will be the values provided (i.e., Provided column names supersede names from first row).
 - If skip_first is False, and no columns are provided, then the column names in the output will be named Col1, Col2, Col3, etc.
 - If skip_first is False, and columns are provided, then the column names in the output will be the values provided.
+- A leading UTF-8 BOM is stripped from the first record, so it neither breaks parsing nor leaks into a column name or value.
 
 ## Configuration Fields
 

internal/pkg/pipeline/task/converter/csv.go

@@ -22,7 +22,13 @@ type csv struct {
 	Columns   []*csvColumn `yaml:"columns" json:"columns"`
 }
 
+var utf8BOM = []byte{0xEF, 0xBB, 0xBF}
+
 func (c *csv) convert(data []byte, _ string) ([]converterOutput, error) {
+	// a leading BOM would otherwise be read as part of the first field, which
+	// csv.Reader rejects when that field is quoted
+	data = bytes.TrimPrefix(data, utf8BOM)
+
 	// Initialize columns if not provided
 	if len(c.Columns) == 0 {
 		if err := c.initializeColumns(data); err != nil {
@@ -80,7 +86,9 @@ func (c *csv) convert(data []byte, _ string) ([]converterOutput, error) {
 
 // initializeColumns sets up column definitions based on the first row of CSV data
 func (c *csv) initializeColumns(data []byte) error {
-	reader := ec.NewReader(bytes.NewReader(data))
+	// padding around the header row is not part of any column name, and whitespace
+	// ahead of a quoted first field would otherwise fail to parse
+	reader := ec.NewReader(bytes.NewReader(bytes.TrimSpace(data)))
 	firstRow, err := reader.Read()
 	if err != nil {
 		return err

test/pipelines/converter/bom_notskip_nocolumns.yaml

@@ -0,0 +1,15 @@
+tasks:
+  - name: pull_sample_csv
+    type: file
+    path: ./test/pipelines/sample_bom.csv
+  - name: split_to_lines
+    type: split
+  - name: convert_from_csv
+    type: converter
+    format: csv
+    skip_first: false
+  - name: join
+    type: join
+  - name: write_sample_csv
+    type: file
+    path: ./test/pipelines/converter/bom_notskip_nocolumns_test_results.json

test/pipelines/converter/bom_notskip_nocolumns_test_results.json

@@ -0,0 +1,5 @@
+{"col1":"Product Name","col2":"SKU (id)","col3":"Unit Price","col4":"In Stock"}
+{"col1":"Widget, Large","col2":"WD-001","col3":"19.99","col4":"true"}
+{"col1":"Bolt \"M8\"","col2":"BL-008","col3":"0.45","col4":"true"}
+{"col1":"Grommet","col2":"GR-100","col3":"2.50","col4":"false"}
+{"col1":"Sprocket, 12T","col2":"SP-012","col3":"7.25","col4":"true"}

test/pipelines/converter/bom_skip_nocolumns.yaml

@@ -0,0 +1,15 @@
+tasks:
+  - name: pull_sample_csv
+    type: file
+    path: ./test/pipelines/sample_bom.csv
+  - name: split_to_lines
+    type: split
+  - name: convert_from_csv
+    type: converter
+    format: csv
+    skip_first: true
+  - name: join
+    type: join
+  - name: write_sample_csv
+    type: file
+    path: ./test/pipelines/converter/bom_skip_nocolumns_test_results.json

test/pipelines/converter/bom_skip_nocolumns_test_results.json

@@ -0,0 +1,4 @@
+{"in_stock":"true","product_name":"Widget, Large","sku_id":"WD-001","unit_price":"19.99"}
+{"in_stock":"true","product_name":"Bolt \"M8\"","sku_id":"BL-008","unit_price":"0.45"}
+{"in_stock":"false","product_name":"Grommet","sku_id":"GR-100","unit_price":"2.50"}
+{"in_stock":"true","product_name":"Sprocket, 12T","sku_id":"SP-012","unit_price":"7.25"}

test/pipelines/sample_bom.csv

@@ -0,0 +1,5 @@
+"Product Name","SKU (id)","Unit Price","In Stock"
+"Widget, Large",WD-001,19.99,true
+"Bolt ""M8""",BL-008,0.45,true
+Grommet,GR-100,2.50,false
+"Sprocket, 12T",SP-012,7.25,true