fix(parser): make Python call extraction linear

.github/workflows/ci.yml

@@ -85,7 +85,7 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
 
       # make codeguard-ci: CI intentionally runs the stable Marketplace action instead of the in-repo binary.
-      - uses: devr-tools/codeguard@v1.2.0
+      - uses: devr-tools/codeguard@a1f8eb3aed6b6b645d42be2a8279b3f578328c40
         with:
           config: .codeguard/codeguard.yaml
 

.github/workflows/release.yml

@@ -232,7 +232,7 @@ jobs:
     # the SLSA generator verifies its own provenance against the tagged ref, and
     # a SHA pin breaks that check. This is the one intentional exception to the
     # SHA-pinning policy.
-    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.1.0
+    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@f7dd8c54c2067bafc12ca7a55595d5ee9b75204a
     with:
       base64-subjects: ${{ needs.build-release.outputs.hashes }}
       upload-assets: true

internal/codeguard/checks/support/python_parser_calls.go

@@ -3,6 +3,7 @@ package support
 import (
 	"regexp"
 	"strings"
+	"unicode"
 )
 
 var pythonCallPattern = regexp.MustCompile(`([A-Za-z_]\w*(?:\s*\.\s*[A-Za-z_]\w*)*)\s*\(`)
@@ -15,8 +16,12 @@ func ExtractCalls(text string, startLine int) []ParsedCall {
 
 // maskedCalls extracts call expressions from masked statement text.
 func maskedCalls(text string, startLine int) []ParsedCall {
-	calls := make([]ParsedCall, 0, 2)
-	for _, match := range pythonCallPattern.FindAllStringSubmatchIndex(text, -1) {
+	matches := pythonCallPattern.FindAllStringSubmatchIndex(text, -1)
+	spans := pythonCallSpans(text, matches)
+	calls := make([]ParsedCall, 0, len(matches))
+	trimmedEnds := make(map[int]int)
+	line, lineOffset := startLine, 0
+	for matchIndex, match := range matches {
 		callee := strings.Join(strings.Fields(strings.ReplaceAll(text[match[2]:match[3]], " .", ".")), "")
 		base := callee
 		if dot := strings.IndexByte(base, '.'); dot >= 0 {
@@ -25,14 +30,86 @@ func maskedCalls(text string, startLine int) []ParsedCall {
 		if isPythonKeyword(base) {
 			continue
 		}
-		open := match[1] - 1
-		args := splitTopLevelArgs(balancedSpan(text, open))
-		line := startLine + strings.Count(text[:match[2]], "\n")
+		line += strings.Count(text[lineOffset:match[2]], "\n")
+		lineOffset = match[2]
+		args := spans[matchIndex].args(text, trimmedEnds)
 		calls = append(calls, ParsedCall{Callee: callee, Args: args, Line: line})
 	}
 	return calls
 }
 
+type pythonCallSpan struct {
+	open   int
+	close  int
+	commas []int
+}
+
+func (span pythonCallSpan) args(text string, trimmedEnds map[int]int) []string {
+	if span.close <= span.open+1 {
+		return nil
+	}
+	args := make([]string, 0, len(span.commas)+1)
+	start := span.open + 1
+	for _, end := range append(span.commas, span.close) {
+		trimmedEnd, ok := trimmedEnds[end]
+		if !ok {
+			trimmedEnd = start + len(strings.TrimRightFunc(text[start:end], unicode.IsSpace))
+			trimmedEnds[end] = trimmedEnd
+		}
+		if trimmedEnd < start {
+			trimmedEnd = start
+		}
+		if arg := strings.TrimLeftFunc(text[start:trimmedEnd], unicode.IsSpace); arg != "" {
+			args = append(args, arg)
+		}
+		start = end + 1
+	}
+	return args
+}
+
+// pythonCallSpans finds the closing parenthesis and top-level commas for all
+// calls in one pass. In particular, it avoids rescanning the remainder of a
+// malformed or deeply nested statement once for every call expression.
+func pythonCallSpans(text string, matches [][]int) []pythonCallSpan {
+	spans := make([]pythonCallSpan, len(matches))
+	callAt := make(map[int]int, len(matches))
+	for index, match := range matches {
+		open := match[1] - 1
+		spans[index] = pythonCallSpan{open: open, close: len(text)}
+		callAt[open] = index
+	}
+
+	type bracket struct {
+		call int
+	}
+	stack := make([]bracket, 0, 8)
+	for offset := 0; offset < len(text); offset++ {
+		switch text[offset] {
+		case '(', '[', '{':
+			call := -1
+			if index, ok := callAt[offset]; ok {
+				call = index
+			}
+			stack = append(stack, bracket{call: call})
+		case ',':
+			if len(stack) > 0 && stack[len(stack)-1].call >= 0 {
+				index := stack[len(stack)-1].call
+				spans[index].commas = append(spans[index].commas, offset)
+			}
+		case ')', ']', '}':
+			if len(stack) == 0 {
+				continue
+			}
+			top := stack[len(stack)-1]
+			stack = stack[:len(stack)-1]
+			if top.call >= 0 {
+				spans[top.call].close = offset
+			}
+		}
+	}
+	return spans
+}
+
 // balancedSpan returns the text between the opening bracket at open and its
 // matching close bracket, exclusive.
 func balancedSpan(text string, open int) string {

tests/support/python_parser_test.go

@@ -138,6 +138,30 @@ func TestParsePythonMultilineCallsAndStatements(t *testing.T) {
 	}
 }
 
+func TestExtractCallsHandlesNestedAndMalformedCalls(t *testing.T) {
+	calls := support.ExtractCalls("outer(inner(value), second)\nnext()", 10)
+	if len(calls) != 3 {
+		t.Fatalf("calls = %+v, want outer, inner, and next", calls)
+	}
+	if calls[0].Callee != "outer" || len(calls[0].Args) != 2 || calls[0].Args[0] != "inner(value)" {
+		t.Fatalf("outer call = %+v", calls[0])
+	}
+	if calls[1].Callee != "inner" || len(calls[1].Args) != 1 || calls[1].Args[0] != "value" {
+		t.Fatalf("inner call = %+v", calls[1])
+	}
+	if calls[2].Line != 11 {
+		t.Fatalf("next line = %d, want 11", calls[2].Line)
+	}
+
+	// Every opening parenthesis used to rescan the rest of this malformed
+	// input, making this small repository-controlled statement quadratic.
+	malformed := strings.Repeat("call(", 20_000)
+	calls = support.ExtractCalls(malformed, 1)
+	if len(calls) != 20_000 {
+		t.Fatalf("malformed calls = %d, want 20000", len(calls))
+	}
+}
+
 func hasImport(imports []support.ParsedImport, module string, alias string) bool {
 	for _, imp := range imports {
 		if imp.Module == module && imp.Alias == alias {