From 9bdde18860010eba06883e99ba279c1d30fee35f Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Fri, 4 Sep 2026 09:40:17 +0200 Subject: [PATCH 1/7] perf(context): Avoid eager case-folded indexes Reuse shared subsystem scoring and scanner inventories directly. Bound prefix and basename routing while preserving uniqueness and ambiguity checks. --- cmd/context_evidence.go | 28 ++ cmd/context_routing.go | 509 ++++++++++++++++++++++---- cmd/context_routing_benchmark_test.go | 114 ++++++ cmd/context_routing_test.go | 144 +++++++- 4 files changed, 714 insertions(+), 81 deletions(-) create mode 100644 cmd/context_routing_benchmark_test.go diff --git a/cmd/context_evidence.go b/cmd/context_evidence.go index 49b4b09..8001456 100644 --- a/cmd/context_evidence.go +++ b/cmd/context_evidence.go @@ -172,6 +172,12 @@ func normalizeContextInventoryPath(file string) string { } func normalizeContextPathWithVolumeGuard(file string, rejectVolume bool) string { + if file == "" || (rejectVolume && isContextVolumePath(file)) { + return "" + } + if isNormalizedContextRelativePath(file) { + return file + } file = strings.ReplaceAll(file, `\`, "/") volumePath := isContextVolumePath(file) file = strings.TrimPrefix(pathpkg.Clean(file), "./") @@ -181,6 +187,28 @@ func normalizeContextPathWithVolumeGuard(file string, rejectVolume bool) string return file } +func isNormalizedContextRelativePath(file string) bool { + if file == "" || file[0] == '/' || file[len(file)-1] == '/' { + return false + } + segmentStart := 0 + for index := 0; index < len(file); index++ { + if file[index] == '\\' { + return false + } + if file[index] != '/' { + continue + } + segment := file[segmentStart:index] + if segment == "" || segment == "." || segment == ".." { + return false + } + segmentStart = index + 1 + } + segment := file[segmentStart:] + return segment != "." && segment != ".." +} + func isContextVolumePath(file string) bool { return strings.HasPrefix(file, "//") || (len(file) >= 2 && file[1] == ':' && ((file[0] >= 'a' && file[0] <= 'z') || (file[0] >= 'A' && file[0] <= 'Z'))) diff --git a/cmd/context_routing.go b/cmd/context_routing.go index f1bf370..620827a 100644 --- a/cmd/context_routing.go +++ b/cmd/context_routing.go @@ -3,6 +3,7 @@ package cmd import ( pathpkg "path" "regexp" + "slices" "sort" "strings" @@ -13,17 +14,20 @@ import ( var contextRoutingTokenPattern = regexp.MustCompile(`[A-Za-z0-9_@./\\-]*[A-Za-z0-9_@]`) type contextFileIndex struct { - caseInsensitive bool - exact map[string][]string - basenames map[string][]string - sortedPaths []contextIndexedPath - prefixPaths map[string][]string + caseInsensitive bool + files []scanner.FileInfo + sortedPaths []contextIndexedPath + useInventory bool + scanInventory bool + normalizeInventorySeparators bool + pathsReady bool } +type contextUniquePathIndex map[string]string + type contextIndexedPath struct { - path string - key string - order int + path string + key string } type contextFileResolution struct { @@ -64,13 +68,17 @@ func resolveContextFileResolutionWithCase(prompt string, files []scanner.FileInf // Exact normalized repository-relative paths always win. for _, token := range tokens { normalized := normalizeContextPath(token) - matches := index.exact[index.key(normalized)] - if normalized != "" && len(matches) == 1 && add(matches[0], false) { + if normalized == "" { + continue + } + match, unique := index.uniqueExact(index.key(normalized)) + if unique && add(match, false) { return resolution } } // Then accept a unique basename that includes its extension. + basenameKeys := make([]string, 0, len(tokens)) for _, token := range tokens { normalized := normalizeContextPath(token) if strings.Contains(normalized, "/") { @@ -80,8 +88,12 @@ func resolveContextFileResolutionWithCase(prompt string, files []scanner.FileInf if pathpkg.Ext(base) == "" { continue } - matches := index.basenames[index.key(base)] - if len(matches) == 1 && add(matches[0], false) { + basenameKeys = append(basenameKeys, index.key(base)) + } + basenameMatches := index.uniqueBasenames(basenameKeys) + for _, baseKey := range basenameKeys { + match, unique := basenameMatches.unique(baseKey) + if unique && add(match, false) { return resolution } } @@ -100,7 +112,7 @@ func resolveContextFileResolutionWithCase(prompt string, files []scanner.FileInf continue } seenPrefixes[prefixKey] = struct{}{} - if index.forPrefix(prefixKey, func(path string) bool { + if index.forPrefix(prefixKey, topK, func(path string) bool { return add(path, true) }) { return resolution @@ -111,92 +123,436 @@ func resolveContextFileResolutionWithCase(prompt string, files []scanner.FileInf } func newContextFileIndex(files []scanner.FileInfo, caseInsensitive bool) contextFileIndex { - index := contextFileIndex{ + return contextFileIndex{ caseInsensitive: caseInsensitive, - exact: make(map[string][]string, len(files)), - basenames: make(map[string][]string), - sortedPaths: make([]contextIndexedPath, 0, len(files)), + files: files, } - for _, file := range files { - path := normalizeContextInventoryPath(file.Path) - if path == "" { +} + +func (i *contextFileIndex) uniqueBasenames(keys []string) contextUniquePathIndex { + if len(keys) == 0 { + return nil + } + matches := make(contextUniquePathIndex, len(keys)) + i.preparePaths() + if len(keys) > 1 && !i.caseInsensitive { + requested := make(map[string]struct{}, len(keys)) + for _, key := range keys { + requested[key] = struct{}{} + } + for index := 0; index < i.pathCount(); index++ { + key := i.basenameKeyAt(index) + if _, ok := requested[key]; ok { + matches.add(key, i.pathAt(index)) + } + } + return matches + } + if len(keys) > 1 { + return i.uniqueCaseFoldedBasenames(keys, matches) + } + for index := 0; index < i.pathCount(); index++ { + if i.basenameMatches(index, keys[0]) { + matches.add(keys[0], i.pathAt(index)) + } + } + return matches +} + +func (i *contextFileIndex) uniqueCaseFoldedBasenames(keys []string, matches contextUniquePathIndex) contextUniquePathIndex { + requested := make(map[uint64]string, len(keys)) + for _, key := range keys { + hash, _ := contextASCIIFoldHash(key) + // Exact comparisons below resolve the unlikely case of a hash collision. + if _, exists := requested[hash]; !exists { + requested[hash] = key + } + } + for index := 0; index < i.pathCount(); index++ { + base := i.basenameAt(index) + hash, ascii := contextASCIIFoldHash(base) + if !ascii { + base = i.key(base) + hash, _ = contextASCIIFoldHash(base) + } + candidate, found := requested[hash] + if !found { continue } - pathKey := index.key(path) - duplicate := false - for _, existing := range index.exact[pathKey] { - if existing == path { - duplicate = true - break + matchesKey := func(key string) bool { + if ascii { + return i.compareKeys(base, key) == 0 } + return base == key } - if duplicate { + if matchesKey(candidate) { + matches.add(candidate, i.pathAt(index)) continue } - index.exact[pathKey] = append(index.exact[pathKey], path) - base := pathpkg.Base(path) - index.basenames[index.key(base)] = append(index.basenames[index.key(base)], path) - index.sortedPaths = append(index.sortedPaths, contextIndexedPath{path: path, key: pathKey}) - } - if caseInsensitive { - sort.Slice(index.sortedPaths, func(i, j int) bool { - return index.sortedPaths[i].path < index.sortedPaths[j].path - }) - for order := range index.sortedPaths { - index.sortedPaths[order].order = order - } - index.prefixPaths = make(map[string][]string) - for _, indexed := range index.sortedPaths { - for prefix := indexed.path; prefix != "."; prefix = pathpkg.Dir(prefix) { - prefixKey := index.key(prefix) - index.prefixPaths[prefixKey] = append(index.prefixPaths[prefixKey], indexed.path) - dir := pathpkg.Dir(prefix) - if dir == prefix { - break - } + for _, key := range keys { + keyHash, _ := contextASCIIFoldHash(key) + if keyHash == hash && matchesKey(key) { + matches.add(key, i.pathAt(index)) + break } } } - sort.Slice(index.sortedPaths, func(i, j int) bool { - if index.sortedPaths[i].key == index.sortedPaths[j].key { - return index.sortedPaths[i].path < index.sortedPaths[j].path + return matches +} + +func (i contextFileIndex) basenameAt(index int) string { + if !i.useInventory && !i.scanInventory { + return pathpkg.Base(i.sortedPaths[index].key) + } + path := i.files[index].Path + if separator := strings.LastIndexAny(path, `/\`); separator >= 0 { + path = path[separator+1:] + } + return path +} + +func (i contextFileIndex) basenameKeyAt(index int) string { + return i.key(i.basenameAt(index)) +} + +func (i contextUniquePathIndex) add(key, path string) { + existing, found := i[key] + if !found { + i[key] = path + return + } + if existing != path { + i[key] = "" + } +} + +func (i contextUniquePathIndex) unique(key string) (string, bool) { + path, found := i[key] + return path, found && path != "" +} + +func (i *contextFileIndex) uniqueExact(key string) (string, bool) { + i.preparePaths() + if i.scanInventory { + match := "" + for index, file := range i.files { + if i.compareKeys(file.Path, key) != 0 { + continue + } + path := i.pathAt(index) + if match != "" && match != path { + return "", false + } + match = path } - return index.sortedPaths[i].key < index.sortedPaths[j].key + return match, match != "" + } + start := sort.Search(i.pathCount(), func(index int) bool { + return i.compareKeyAt(index, key) >= 0 }) - return index + if start == i.pathCount() || i.compareKeyAt(start, key) != 0 { + return "", false + } + path := i.pathAt(start) + if start+1 < i.pathCount() && i.compareKeyAt(start+1, key) == 0 && i.pathAt(start+1) != path { + return "", false + } + return path, true } -func (i contextFileIndex) forPrefix(prefixKey string, visit func(string) bool) bool { - if i.caseInsensitive { - for _, path := range i.prefixPaths[prefixKey] { - if visit(path) { +func (i *contextFileIndex) preparePaths() { + if i.pathsReady { + return + } + // Binary-search ordered inventories; scan valid unordered inventories directly. + inventoryValid := true + inventoryOrdered := true + for index, file := range i.files { + valid, normalizeSeparators := contextInventoryPathState(file.Path) + i.normalizeInventorySeparators = i.normalizeInventorySeparators || normalizeSeparators + if !valid { + inventoryValid = false + break + } + if inventoryOrdered && index > 0 && i.comparePaths(i.files[index-1].Path, file.Path) >= 0 { + inventoryOrdered = false + } + } + i.useInventory = inventoryValid && inventoryOrdered + if i.useInventory { + i.pathsReady = true + return + } + if inventoryValid { + i.scanInventory = true + i.pathsReady = true + return + } + + i.sortedPaths = make([]contextIndexedPath, 0, len(i.files)) + for _, file := range i.files { + path := normalizeContextInventoryPath(file.Path) + if path != "" { + i.sortedPaths = append(i.sortedPaths, contextIndexedPath{path: path, key: i.key(path)}) + } + } + compare := func(left, right contextIndexedPath) int { + if order := strings.Compare(left.key, right.key); order != 0 { + return order + } + return strings.Compare(left.path, right.path) + } + slices.SortFunc(i.sortedPaths, compare) + write := min(1, len(i.sortedPaths)) + for read := 1; read < len(i.sortedPaths); read++ { + if i.sortedPaths[read] == i.sortedPaths[write-1] { + continue + } + if write != read { + i.sortedPaths[write] = i.sortedPaths[read] + } + write++ + } + i.sortedPaths = i.sortedPaths[:write] + i.pathsReady = true +} + +func (i *contextFileIndex) forPrefix(prefixKey string, limit int, visit func(string) bool) bool { + if limit <= 0 { + return false + } + i.preparePaths() + matches := make([]int, 0, min(limit, i.pathCount())) + consider := func(candidate int) { + position := sort.Search(len(matches), func(index int) bool { + return i.compareOutputPaths(matches[index], candidate) >= 0 + }) + if position < len(matches) && i.compareOutputPaths(matches[position], candidate) == 0 { + return + } + if len(matches) == limit && position == len(matches) { + return + } + if len(matches) < limit { + matches = append(matches, 0) + } + copy(matches[position+1:], matches[position:len(matches)-1]) + matches[position] = candidate + } + visitMatches := func() bool { + for _, match := range matches { + if visit(i.pathAt(match)) { return true } } return false } - - exactStart := sort.Search(len(i.sortedPaths), func(index int) bool { - return i.sortedPaths[index].key >= prefixKey + descendantPrefix := prefixKey + "/" + if i.scanInventory { + for index, file := range i.files { + if i.compareKeys(file.Path, prefixKey) == 0 || contextPathPrefix(file.Path, descendantPrefix, i.caseInsensitive) { + consider(index) + } + } + return visitMatches() + } + exactStart := sort.Search(i.pathCount(), func(index int) bool { + return i.compareKeyAt(index, prefixKey) >= 0 }) - for index := exactStart; index < len(i.sortedPaths) && i.sortedPaths[index].key == prefixKey; index++ { - if visit(i.sortedPaths[index].path) { + descendantStart := sort.Search(i.pathCount(), func(index int) bool { + return i.compareKeyAt(index, descendantPrefix) >= 0 + }) + if i.caseInsensitive { + for index := exactStart; index < i.pathCount() && i.compareKeyAt(index, prefixKey) == 0; index++ { + consider(index) + } + for index := descendantStart; index < i.pathCount() && i.keyHasPrefixAt(index, descendantPrefix); index++ { + consider(index) + } + return visitMatches() + } + + visited := 0 + for index := exactStart; index < i.pathCount() && i.compareKeyAt(index, prefixKey) == 0; index++ { + if visited == limit { + return false + } + visited++ + if visit(i.pathAt(index)) { return true } } - descendantPrefix := prefixKey + "/" - descendantStart := sort.Search(len(i.sortedPaths), func(index int) bool { - return i.sortedPaths[index].key >= descendantPrefix - }) - for index := descendantStart; index < len(i.sortedPaths) && strings.HasPrefix(i.sortedPaths[index].key, descendantPrefix); index++ { - if visit(i.sortedPaths[index].path) { + for index := descendantStart; index < i.pathCount() && i.keyHasPrefixAt(index, descendantPrefix); index++ { + if visited == limit { + return false + } + visited++ + if visit(i.pathAt(index)) { return true } } return false } +func (i contextFileIndex) pathCount() int { + if i.useInventory || i.scanInventory { + return len(i.files) + } + return len(i.sortedPaths) +} + +func (i contextFileIndex) pathAt(index int) string { + if i.useInventory || i.scanInventory { + path := i.files[index].Path + if i.normalizeInventorySeparators { + return strings.ReplaceAll(path, `\`, "/") + } + return path + } + return i.sortedPaths[index].path +} + +func (i contextFileIndex) compareOutputPaths(left, right int) int { + if i.useInventory || i.scanInventory { + if !i.normalizeInventorySeparators { + return strings.Compare(i.files[left].Path, i.files[right].Path) + } + return compareContextPaths(i.files[left].Path, i.files[right].Path, false) + } + return strings.Compare(i.sortedPaths[left].path, i.sortedPaths[right].path) +} + +func (i contextFileIndex) compareKeyAt(index int, key string) int { + if i.useInventory { + return i.compareKeys(i.files[index].Path, key) + } + return strings.Compare(i.sortedPaths[index].key, key) +} + +func (i contextFileIndex) keyHasPrefixAt(index int, prefix string) bool { + if i.useInventory { + if !i.normalizeInventorySeparators && !i.caseInsensitive { + return strings.HasPrefix(i.files[index].Path, prefix) + } + return contextPathPrefix(i.files[index].Path, prefix, i.caseInsensitive) + } + return strings.HasPrefix(i.sortedPaths[index].key, prefix) +} + +func (i contextFileIndex) basenameMatches(index int, key string) bool { + if i.useInventory || i.scanInventory { + path := i.files[index].Path + if separator := strings.LastIndexAny(path, `/\`); separator >= 0 { + path = path[separator+1:] + } + return i.compareKeys(path, key) == 0 + } + return pathpkg.Base(i.sortedPaths[index].key) == key +} + +func (i contextFileIndex) comparePaths(left, right string) int { + if order := i.compareKeys(left, right); order != 0 { + return order + } + return compareContextPaths(left, right, false) +} + +func (i contextFileIndex) compareKeys(left, right string) int { + if !i.normalizeInventorySeparators && !i.caseInsensitive { + return strings.Compare(left, right) + } + return compareContextPaths(left, right, i.caseInsensitive) +} + +func compareContextPaths(left, right string, caseInsensitive bool) int { + limit := min(len(left), len(right)) + for index := 0; index < limit; index++ { + leftByte, rightByte := left[index], right[index] + if leftByte == '\\' { + leftByte = '/' + } + if rightByte == '\\' { + rightByte = '/' + } + if leftByte >= 0x80 || rightByte >= 0x80 { + return strings.Compare(contextComparisonKey(left, caseInsensitive), contextComparisonKey(right, caseInsensitive)) + } + if leftByte == rightByte { + continue + } + if caseInsensitive && leftByte >= 'A' && leftByte <= 'Z' { + leftByte += 'a' - 'A' + } + if caseInsensitive && rightByte >= 'A' && rightByte <= 'Z' { + rightByte += 'a' - 'A' + } + if leftByte < rightByte { + return -1 + } + if leftByte > rightByte { + return 1 + } + } + return len(left) - len(right) +} + +func contextPathPrefix(path, prefix string, caseInsensitive bool) bool { + if len(path) < len(prefix) { + return strings.HasPrefix(contextComparisonKey(path, caseInsensitive), contextComparisonKey(prefix, caseInsensitive)) + } + for index := range prefix { + pathByte, prefixByte := path[index], prefix[index] + if pathByte == '\\' { + pathByte = '/' + } + if prefixByte == '\\' { + prefixByte = '/' + } + if pathByte >= 0x80 || prefixByte >= 0x80 { + return strings.HasPrefix(contextComparisonKey(path, caseInsensitive), contextComparisonKey(prefix, caseInsensitive)) + } + if caseInsensitive && pathByte >= 'A' && pathByte <= 'Z' { + pathByte += 'a' - 'A' + } + if caseInsensitive && prefixByte >= 'A' && prefixByte <= 'Z' { + prefixByte += 'a' - 'A' + } + if pathByte != prefixByte { + return false + } + } + return true +} + +func contextComparisonKey(path string, caseInsensitive bool) string { + path = strings.ReplaceAll(path, `\`, "/") + if caseInsensitive { + path = strings.ToLower(path) + } + return path +} + +func contextInventoryPathState(path string) (valid, normalizeSeparators bool) { + if path == "" || path[0] == '/' || path[0] == '\\' || path[len(path)-1] == '/' || path[len(path)-1] == '\\' { + return false, false + } + segmentStart := 0 + for index := 0; index < len(path); index++ { + if path[index] != '/' && path[index] != '\\' { + continue + } + normalizeSeparators = normalizeSeparators || path[index] == '\\' + segment := path[segmentStart:index] + if segment == "" || segment == "." || segment == ".." { + return false, normalizeSeparators + } + segmentStart = index + 1 + } + segment := path[segmentStart:] + return segment != "." && segment != "..", normalizeSeparators +} + func (i contextFileIndex) key(value string) string { if i.caseInsensitive { return strings.ToLower(value) @@ -204,6 +560,23 @@ func (i contextFileIndex) key(value string) string { return value } +func contextASCIIFoldHash(value string) (uint64, bool) { + const offset64 = 14695981039346656037 + const prime64 = 1099511628211 + hash := uint64(offset64) + ascii := true + for index := 0; index < len(value); index++ { + char := value[index] + ascii = ascii && char < 0x80 + if char >= 'A' && char <= 'Z' { + char += 'a' - 'A' + } + hash ^= uint64(char) + hash *= prime64 + } + return hash, ascii +} + func contextRoutingTokens(prompt string) []string { matches := contextRoutingTokenPattern.FindAllString(strings.ReplaceAll(prompt, `\`, "/"), -1) tokens := make([]string, 0, len(matches)) diff --git a/cmd/context_routing_benchmark_test.go b/cmd/context_routing_benchmark_test.go new file mode 100644 index 0000000..9590189 --- /dev/null +++ b/cmd/context_routing_benchmark_test.go @@ -0,0 +1,114 @@ +package cmd + +import ( + "fmt" + "sort" + "strings" + "testing" + + "codemap/scanner" +) + +func benchmarkContextRoutingFiles() []scanner.FileInfo { + files := make([]string, 50_000) + for i := range files { + files[i] = fmt.Sprintf("Root/Area%02d/Feature%03d/Package%03d/file%05d.go", i%40, i%200, i%500, i) + } + sort.Strings(files) + return routingFiles(files...) +} + +func BenchmarkContextFileIndexCaseInsensitive(b *testing.B) { + routing := benchmarkContextRoutingFiles() + for _, benchmark := range []struct { + name string + prefix string + }{ + {name: "broad prefix", prefix: "root"}, + {name: "narrow prefix", prefix: "root/area39/feature199"}, + } { + b.Run(benchmark.name, func(b *testing.B) { + b.ReportAllocs() + for range b.N { + index := newContextFileIndex(routing, true) + index.forPrefix(benchmark.prefix, 3, func(string) bool { return false }) + } + }) + } + b.Run("exact", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + index := newContextFileIndex(routing, true) + _, _ = index.uniqueExact(index.key("Root/Area39/Feature199/Package499/file49999.go")) + } + }) + b.Run("basename", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + index := newContextFileIndex(routing, true) + matches := index.uniqueBasenames([]string{index.key("file49999.go")}) + _, _ = matches.unique(index.key("file49999.go")) + } + }) +} + +func BenchmarkContextFileIndexCaseInsensitiveMixedCase(b *testing.B) { + files := make([]string, 50_000) + for i := range files { + root := "Root" + if i%2 == 0 { + root = "alpha" + } + files[i] = fmt.Sprintf("%s/Area%02d/Feature%03d/file%05d.go", root, i%40, i%200, i) + } + sort.Strings(files) + routing := routingFiles(files...) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + index := newContextFileIndex(routing, true) + index.forPrefix("root", 3, func(string) bool { return false }) + } +} + +func BenchmarkContextFileIndexCaseInsensitiveWindowsPaths(b *testing.B) { + routing := benchmarkContextRoutingFiles() + for i := range routing { + routing[i].Path = strings.ReplaceAll(routing[i].Path, "/", `\`) + } + probe := newContextFileIndex(routing, true) + probe.preparePaths() + if !probe.useInventory { + b.Fatal("ordered backslash-separated inventory used fallback index") + } + b.ReportAllocs() + b.ResetTimer() + for range b.N { + index := newContextFileIndex(routing, true) + index.forPrefix("root", 3, func(string) bool { return false }) + } +} + +func BenchmarkContextFileIndexCaseInsensitiveWindowsBasename(b *testing.B) { + routing := benchmarkContextRoutingFiles() + for i := range routing { + routing[i].Path = strings.ReplaceAll(routing[i].Path, "/", `\`) + } + b.ReportAllocs() + b.ResetTimer() + for range b.N { + index := newContextFileIndex(routing, true) + matches := index.uniqueBasenames([]string{"file49999.go"}) + _, _ = matches.unique("file49999.go") + } +} + +func BenchmarkContextFileIndexCaseSensitive(b *testing.B) { + routing := benchmarkContextRoutingFiles() + b.ReportAllocs() + b.ResetTimer() + for range b.N { + index := newContextFileIndex(routing, false) + index.forPrefix("Root", 3, func(string) bool { return false }) + } +} diff --git a/cmd/context_routing_test.go b/cmd/context_routing_test.go index 14dc340..840879e 100644 --- a/cmd/context_routing_test.go +++ b/cmd/context_routing_test.go @@ -62,7 +62,7 @@ func TestContextLexicalRouting(t *testing.T) { }) t.Run("normalized duplicates resolve once", func(t *testing.T) { - files := routingFiles("./cmd/context.go", "cmd/context.go", "internal/context.go") + files := routingFiles("./cmd/context.go", "cmd//context.go", "cmd/context.go/", "cmd/context.go", "internal/context.go") got := resolveContextFilesWithCase("inspect cmd/./context.go and context", files, config.ProjectConfig{}, 2, false) if want := []string{"cmd/context.go"}; !reflect.DeepEqual(got, want) { t.Fatalf("normalized files = %#v, want %#v", got, want) @@ -95,7 +95,7 @@ func TestContextLexicalRouting(t *testing.T) { }) t.Run("case-folded exact collisions stay unresolved", func(t *testing.T) { - files := routingFiles("Cmd/Foo.go", "cmd/foo.go") + files := routingFiles("Cmd/Foo.go", "Root/a.go", "alpha/b.go", "cmd/foo.go") got := resolveContextFilesWithCase(`inspect CMD\FOO.GO`, files, config.ProjectConfig{}, 2, true) if len(got) != 0 { t.Fatalf("case-collision files = %#v, want none", got) @@ -146,6 +146,17 @@ func TestContextLexicalRouting(t *testing.T) { } }) + t.Run("case-folded subsystem route fills after an explicit match", func(t *testing.T) { + files := routingFiles("src/build/c.go", "SRC/build/a.go", "Src/Build/b.go") + cfg := config.ProjectConfig{Routing: config.RoutingConfig{ + Subsystems: []config.Subsystem{{ID: "build", Keywords: []string{"overdrive"}, Paths: []string{"src/build"}}}, + }} + got := resolveContextFilesWithCase("inspect SRC/build/a.go during overdrive", files, cfg, 2, true) + if want := []string{"SRC/build/a.go", "Src/Build/b.go"}; !reflect.DeepEqual(got, want) { + t.Fatalf("files = %#v, want %#v", got, want) + } + }) + t.Run("explicit basename drives intent risk", func(t *testing.T) { files := routingFiles("src/final_build.rs", "a.rs", "b.rs", "c.rs") graph := &scanner.FileGraph{ @@ -206,47 +217,154 @@ func routingFiles(paths ...string) []scanner.FileInfo { func TestContextFileIndexPrefixes(t *testing.T) { index := newContextFileIndex(routingFiles("src/build/z.go", "src/build/a.go", "src/other.go"), false) var got []string - index.forPrefix("src/build", func(path string) bool { + index.forPrefix("src/build", 1, func(path string) bool { got = append(got, path) return false }) - if want := []string{"src/build/a.go", "src/build/z.go"}; !reflect.DeepEqual(got, want) { + if want := []string{"src/build/a.go"}; !reflect.DeepEqual(got, want) { t.Fatalf("prefix files = %#v, want %#v", got, want) } } +func TestContextFileIndexReusesOrderedInventory(t *testing.T) { + index := newContextFileIndex(routingFiles("Root/a.go", "Root/b.go"), true) + index.preparePaths() + if !index.useInventory { + t.Fatal("ordered inventory used fallback index") + } + + index = newContextFileIndex(routingFiles("Root/a.go", "alpha/b.go"), true) + index.preparePaths() + if index.useInventory { + t.Fatal("unordered case-folded inventory used binary search") + } + if len(index.sortedPaths) != 0 { + t.Fatal("unordered case-folded inventory built a fallback index") + } + match, unique := index.uniqueExact("alpha/b.go") + if !unique || match != "alpha/b.go" { + t.Fatalf("direct-scan exact match = %q, %v", match, unique) + } + + index = newContextFileIndex(routingFiles(`Root\a.go`, `Root\b.go`), true) + index.preparePaths() + if !index.useInventory { + t.Fatal("ordered backslash-separated inventory used fallback index") + } + match, unique = index.uniqueExact("root/a.go") + if !unique || match != "Root/a.go" { + t.Fatalf("backslash-separated exact match = %q, %v", match, unique) + } + var prefixed []string + index.forPrefix("root", 2, func(path string) bool { + prefixed = append(prefixed, path) + return false + }) + if want := []string{"Root/a.go", "Root/b.go"}; !reflect.DeepEqual(prefixed, want) { + t.Fatalf("backslash-separated prefix matches = %#v, want %#v", prefixed, want) + } + basenames := index.uniqueBasenames([]string{"b.go"}) + if match, unique = basenames.unique("b.go"); !unique || match != "Root/b.go" { + t.Fatalf("backslash-separated basename match = %q, %v", match, unique) + } + + index = newContextFileIndex(routingFiles("Root/a.go", `Root\a.go`), true) + index.preparePaths() + if index.useInventory { + t.Fatal("separator-normalized duplicate reused inventory") + } +} + func TestContextFileIndexPrefixesRespectBoundaries(t *testing.T) { - index := newContextFileIndex(routingFiles("src/build.go", "src/build/a.go", "src/building/b.go"), false) + index := newContextFileIndex(routingFiles("src/build.go", "src/build", "src/build/a.go", "src/building/b.go"), false) var got []string - index.forPrefix("src/build", func(path string) bool { + index.forPrefix("src/build", 10, func(path string) bool { got = append(got, path) return false }) - if want := []string{"src/build/a.go"}; !reflect.DeepEqual(got, want) { + if want := []string{"src/build", "src/build/a.go"}; !reflect.DeepEqual(got, want) { t.Fatalf("boundary files = %#v, want %#v", got, want) } } +func TestContextFileIndexPrefixLimitCountsUniquePaths(t *testing.T) { + index := newContextFileIndex(routingFiles("src/a.go", `src\a.go`, "src/b.go"), false) + var got []string + index.forPrefix("src", 2, func(path string) bool { + got = append(got, path) + return false + }) + if want := []string{"src/a.go", "src/b.go"}; !reflect.DeepEqual(got, want) { + t.Fatalf("prefix files = %#v, want %#v", got, want) + } +} + +func TestContextFileIndexBasenamesUseLargeKeySet(t *testing.T) { + for _, test := range []struct { + name string + files []scanner.FileInfo + caseInsensitive bool + want string + }{ + {name: "ordered", files: routingFiles("other/a.go", "pkg/a.go", "pkg/b.go", "pkg/c.go", "pkg/d.go", "pkg/e.go"), want: "pkg/e.go"}, + {name: "unordered", files: routingFiles("pkg/e.go", "pkg/d.go", "pkg/c.go", "pkg/b.go", "pkg/a.go", "other/a.go"), want: "pkg/e.go"}, + {name: "normalized fallback", files: routingFiles("./pkg/e.go", "pkg/d.go", "pkg/c.go", "pkg/b.go", "pkg/a.go", "other/a.go"), want: "pkg/e.go"}, + {name: "case-folded separators", files: routingFiles(`Other\a.go`, `Pkg\a.go`, `Pkg\b.go`, `Pkg\c.go`, `Pkg\d.go`, `Pkg\e.go`), caseInsensitive: true, want: "Pkg/e.go"}, + } { + t.Run(test.name, func(t *testing.T) { + index := newContextFileIndex(test.files, test.caseInsensitive) + keys := []string{"a.go", "b.go", "c.go", "d.go", "e.go"} + for keyIndex := range keys { + keys[keyIndex] = index.key(keys[keyIndex]) + } + matches := index.uniqueBasenames(keys) + if _, unique := matches.unique(index.key("a.go")); unique { + t.Fatal("ambiguous basename resolved") + } + if got, unique := matches.unique(index.key("e.go")); !unique || got != test.want { + t.Fatalf("unique basename = %q, %v, want %s, true", got, unique, test.want) + } + }) + } + + index := newContextFileIndex(routingFiles("pkg/Ä.go", "pkg/b.go", "pkg/c.go", "pkg/d.go", "pkg/e.go"), true) + keys := []string{index.key("ä.go"), "b.go", "c.go", "d.go", "e.go"} + matches := index.uniqueBasenames(keys) + if got, unique := matches.unique(index.key("ä.go")); !unique || got != "pkg/Ä.go" { + t.Fatalf("Unicode basename = %q, %v, want pkg/Ä.go, true", got, unique) + } +} + func TestContextFileIndexPrefixesRespectCaseFolding(t *testing.T) { - files := routingFiles("Src/Build/z.go", "src/build/a.go", "src/building/b.go") + files := routingFiles("src/Build/d.go", "Src/build/a.go", "SRC/BUILD/c.go", "src/build/b.go", "src/building/e.go") index := newContextFileIndex(files, true) - if want := []string{"Src/Build/z.go", "src/build/a.go"}; !reflect.DeepEqual(index.prefixPaths["src/build"], want) { - t.Fatalf("case-folded prefix index = %#v, want %#v", index.prefixPaths["src/build"], want) + if index.pathsReady { + t.Fatal("case-folded inventory prepared eagerly") } var got []string - index.forPrefix(index.key("src/build"), func(path string) bool { + index.forPrefix(index.key("src/build"), 2, func(path string) bool { got = append(got, path) return false }) - if want := []string{"Src/Build/z.go", "src/build/a.go"}; !reflect.DeepEqual(got, want) { + if want := []string{"SRC/BUILD/c.go", "Src/build/a.go"}; !reflect.DeepEqual(got, want) { t.Fatalf("case-folded files = %#v, want %#v", got, want) } + if !index.pathsReady { + t.Fatal("case-folded inventory was not prepared on demand") + } cfg := config.ProjectConfig{Routing: config.RoutingConfig{ Subsystems: []config.Subsystem{{ID: "build", Keywords: []string{"build"}, Paths: []string{"src/build"}}}, }} - if got := resolveContextFilesWithCase("build", files, cfg, 1, true); !reflect.DeepEqual(got, []string{"Src/Build/z.go"}) { + if got := resolveContextFilesWithCase("build", files, cfg, 1, true); !reflect.DeepEqual(got, []string{"SRC/BUILD/c.go"}) { t.Fatalf("case-folded top-k files = %#v, want first path", got) } + + index = newContextFileIndex(routingFiles("Ärea/a.go", "ärea/b.go"), true) + got = nil + index.forPrefix(index.key("ÄREA"), 2, func(path string) bool { got = append(got, path); return false }) + if want := []string{"Ärea/a.go", "ärea/b.go"}; !reflect.DeepEqual(got, want) { + t.Fatalf("Unicode case-folded files = %#v, want %#v", got, want) + } } func TestContextSubsystemMatchesUsesSharedRouteScoring(t *testing.T) { From fa6331266e21a820c27db099bc0890df8392ea64 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Fri, 4 Sep 2026 09:40:24 +0200 Subject: [PATCH 2/7] test(scanner): Isolate Rust graph fixtures Keep Rust coverage and workspace-boundary fixtures independent of Cargo metadata latency. Preserve parser and graph contracts in focused tests. --- mcp/main_more_test.go | 1 - scanner/filegraph_test.go | 26 ++++++++++++++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/mcp/main_more_test.go b/mcp/main_more_test.go index 56cc6c6..199e3d2 100644 --- a/mcp/main_more_test.go +++ b/mcp/main_more_test.go @@ -451,7 +451,6 @@ func TestRustGraphContextHandlersDisclosePartialCoverage(t *testing.T) { root := t.TempDir() files := map[string]string{ - "Cargo.toml": "[package]\nname = \"demo\"\nversion = \"0.1.0\"\n", "src/lib.rs": "mod workspace;\n", "src/workspace.rs": "pub fn run() {}\n", "src/string_route.rs": "const COMMAND: &str = \"run\";\n", diff --git a/scanner/filegraph_test.go b/scanner/filegraph_test.go index 9f31c13..3052a1b 100644 --- a/scanner/filegraph_test.go +++ b/scanner/filegraph_test.go @@ -13,10 +13,6 @@ import ( ) func TestRustWorkspaceImportersRespectCrateBoundaries(t *testing.T) { - if !NewAstGrepAnalyzer().Available() { - t.Skip("ast-grep not available") - } - root := t.TempDir() files := map[string]string{ "Cargo.toml": `[workspace] @@ -57,9 +53,27 @@ but-api = { path = "../but-api" } } } - graph, err := BuildFileGraph(context.Background(), root, ConfiguredFilters(root)) + analyses := []FileAnalysis{ + {Path: "crate-a/src/lib.rs", Language: "rust", References: []ImportReference{{Path: "workspace", Kind: "rust-module"}}}, + {Path: "crate-a/src/workspace.rs", Language: "rust"}, + {Path: "but-api/src/lib.rs", Language: "rust", References: []ImportReference{{Path: "workspace", Kind: "rust-module"}}}, + {Path: "but-api/src/workspace.rs", Language: "rust"}, + {Path: "consumer/src/lib.rs", Language: "rust", References: []ImportReference{{Path: "but_api::workspace::run", Kind: "rust-path"}}}, + } + metadata := cargoMetadataJSON(t, root, []map[string]any{ + cargoPackage(root, "crate-a", "crate-a", "crate_a", nil), + cargoPackage(root, "but-api", "but-api", "but_api", nil), + cargoPackage(root, "consumer", "consumer", "consumer", []map[string]any{{ + "name": "but-api", + "path": filepath.Join(root, "but-api"), + }}), + }) + graph, err := buildFileGraphFromAnalysesWithCargoMetadata( + context.Background(), root, analyses, + func(context.Context, string) ([]byte, error) { return metadata, nil }, + ) if err != nil { - t.Fatalf("BuildFileGraph() error: %v", err) + t.Fatalf("buildFileGraphFromAnalysesWithCargoMetadata() error: %v", err) } want := []string{"but-api/src/lib.rs", "consumer/src/lib.rs"} From 1ed95f25a684effde0d90b87627411271bbad38f Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:21:17 +0200 Subject: [PATCH 3/7] test(perf): Cover repository-scale hot paths Measure routing, scanner indexing, topology discovery, rendering, and watch publication with deterministic large inventories. --- cmd/context_routing_benchmark_test.go | 77 +++++++++++++-- render/hotpath_benchmark_test.go | 53 ++++++++++ scanner/hotpath_benchmark_test.go | 134 ++++++++++++++++++++++++++ topology/provider_benchmark_test.go | 90 +++++++++++++++++ watch/publication_benchmark_test.go | 67 +++++++++++++ 5 files changed, 415 insertions(+), 6 deletions(-) create mode 100644 render/hotpath_benchmark_test.go create mode 100644 scanner/hotpath_benchmark_test.go create mode 100644 topology/provider_benchmark_test.go create mode 100644 watch/publication_benchmark_test.go diff --git a/cmd/context_routing_benchmark_test.go b/cmd/context_routing_benchmark_test.go index 9590189..5f8fb50 100644 --- a/cmd/context_routing_benchmark_test.go +++ b/cmd/context_routing_benchmark_test.go @@ -1,16 +1,21 @@ package cmd import ( + "context" "fmt" "sort" "strings" "testing" + "codemap/analysis" "codemap/scanner" ) -func benchmarkContextRoutingFiles() []scanner.FileInfo { - files := make([]string, 50_000) +var benchmarkContextEnvelope ContextEnvelope +var benchmarkContextMatches contextUniquePathIndex + +func benchmarkContextRoutingFiles(count int) []scanner.FileInfo { + files := make([]string, count) for i := range files { files[i] = fmt.Sprintf("Root/Area%02d/Feature%03d/Package%03d/file%05d.go", i%40, i%200, i%500, i) } @@ -19,7 +24,7 @@ func benchmarkContextRoutingFiles() []scanner.FileInfo { } func BenchmarkContextFileIndexCaseInsensitive(b *testing.B) { - routing := benchmarkContextRoutingFiles() + routing := benchmarkContextRoutingFiles(50_000) for _, benchmark := range []struct { name string prefix string @@ -72,7 +77,7 @@ func BenchmarkContextFileIndexCaseInsensitiveMixedCase(b *testing.B) { } func BenchmarkContextFileIndexCaseInsensitiveWindowsPaths(b *testing.B) { - routing := benchmarkContextRoutingFiles() + routing := benchmarkContextRoutingFiles(50_000) for i := range routing { routing[i].Path = strings.ReplaceAll(routing[i].Path, "/", `\`) } @@ -90,7 +95,7 @@ func BenchmarkContextFileIndexCaseInsensitiveWindowsPaths(b *testing.B) { } func BenchmarkContextFileIndexCaseInsensitiveWindowsBasename(b *testing.B) { - routing := benchmarkContextRoutingFiles() + routing := benchmarkContextRoutingFiles(50_000) for i := range routing { routing[i].Path = strings.ReplaceAll(routing[i].Path, "/", `\`) } @@ -103,8 +108,37 @@ func BenchmarkContextFileIndexCaseInsensitiveWindowsBasename(b *testing.B) { } } +func BenchmarkContextFileIndexManyBasenames(b *testing.B) { + keys := make([]string, 128) + for index := range keys { + keys[index] = fmt.Sprintf("file%05d.go", index*317) + } + for _, benchmark := range []struct { + name string + caseInsensitive bool + }{ + {name: "case sensitive"}, + {name: "case folded", caseInsensitive: true}, + } { + b.Run(benchmark.name, func(b *testing.B) { + routing := benchmarkContextRoutingFiles(50_000) + if benchmark.caseInsensitive { + for index := range routing { + routing[index].Path = strings.Replace(routing[index].Path, "/file", "/FILE", 1) + } + } + b.ReportAllocs() + b.ResetTimer() + for range b.N { + index := newContextFileIndex(routing, benchmark.caseInsensitive) + benchmarkContextMatches = index.uniqueBasenames(keys) + } + }) + } +} + func BenchmarkContextFileIndexCaseSensitive(b *testing.B) { - routing := benchmarkContextRoutingFiles() + routing := benchmarkContextRoutingFiles(50_000) b.ReportAllocs() b.ResetTimer() for range b.N { @@ -112,3 +146,34 @@ func BenchmarkContextFileIndexCaseSensitive(b *testing.B) { index.forPrefix("Root", 3, func(string) bool { return false }) } } + +func BenchmarkBuildContextEnvelope(b *testing.B) { + files := benchmarkContextRoutingFiles(5_000) + imports := make(map[string][]string, len(files)) + importers := make(map[string][]string, len(files)) + for i := 1; i < len(files); i++ { + current := files[i].Path + previous := files[i-1].Path + imports[current] = []string{previous} + importers[previous] = []string{current} + } + graph := &scanner.FileGraph{ + Imports: imports, + Importers: importers, + Coverage: scanner.GraphCoverage{Status: analysis.CoverageComplete}, + } + deps := testContextEnvelopeDeps(files, graph) + root := b.TempDir() + + b.ReportAllocs() + b.ResetTimer() + for range b.N { + benchmarkContextEnvelope = buildContextEnvelopeWithDeps( + context.Background(), + root, + "refactor Root/Area39/Feature199/Package499/file49999.go", + true, + deps, + ) + } +} diff --git a/render/hotpath_benchmark_test.go b/render/hotpath_benchmark_test.go new file mode 100644 index 0000000..6dde02a --- /dev/null +++ b/render/hotpath_benchmark_test.go @@ -0,0 +1,53 @@ +package render + +import ( + "fmt" + "io" + "path/filepath" + "testing" + + "codemap/scanner" +) + +var benchmarkTree *treeNode + +func BenchmarkBuildTreeStructure(b *testing.B) { + files := benchmarkRenderFiles(50_000) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + benchmarkTree = buildTreeStructure(files) + } +} + +func BenchmarkTree(b *testing.B) { + project := scanner.Project{Root: "benchmark", Files: benchmarkRenderFiles(5_000)} + b.ReportAllocs() + b.ResetTimer() + for range b.N { + Tree(io.Discard, project) + } +} + +func BenchmarkSkyline(b *testing.B) { + project := scanner.Project{Root: "benchmark", Files: benchmarkRenderFiles(50_000)} + b.ReportAllocs() + b.ResetTimer() + for range b.N { + Skyline(io.Discard, project, false) + } +} + +func benchmarkRenderFiles(count int) []scanner.FileInfo { + files := make([]scanner.FileInfo, count) + extensions := []string{".go", ".ts", ".py", ".rs", ".java"} + for i := range files { + ext := extensions[i%len(extensions)] + files[i] = scanner.FileInfo{ + Path: filepath.Join(fmt.Sprintf("area-%03d", i%100), fmt.Sprintf("feature-%03d", i%500), fmt.Sprintf("file-%05d%s", i, ext)), + Size: int64(64 + i%4096), + Ext: ext, + } + } + return files +} diff --git a/scanner/hotpath_benchmark_test.go b/scanner/hotpath_benchmark_test.go new file mode 100644 index 0000000..9a5925f --- /dev/null +++ b/scanner/hotpath_benchmark_test.go @@ -0,0 +1,134 @@ +package scanner + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" +) + +var ( + benchmarkScannedFiles []FileInfo + benchmarkFileIndex *fileIndex + benchmarkGraph *FileGraph + benchmarkResolved []string +) + +func BenchmarkScanFiles(b *testing.B) { + root, _ := benchmarkScannerTree(b, 5_000) + cache := NewGitIgnoreCache(root) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + var err error + benchmarkScannedFiles, err = ScanFiles(context.Background(), root, cache, nil, nil) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkBuildFileGraphFromAnalyses(b *testing.B) { + root, files := benchmarkScannerTree(b, 5_000) + analyses := make([]FileAnalysis, 0, len(files)) + for _, file := range files { + analyses = append(analyses, FileAnalysis{ + Path: file.Path, + Language: "go", + Imports: []string{"example.com/bench/shared"}, + }) + } + b.ReportAllocs() + b.ResetTimer() + for range b.N { + var err error + benchmarkGraph, err = BuildFileGraphFromAnalyses(context.Background(), root, analyses, Filters{}) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkBuildFileIndex(b *testing.B) { + files := benchmarkFileInventory(50_000) + benchmarkBuildFileIndex(b, files) +} + +func BenchmarkBuildFileIndexSparseDirectories(b *testing.B) { + files := make([]FileInfo, 50_000) + for i := range files { + files[i] = FileInfo{ + Path: fmt.Sprintf("pkg/area-%05d/file.go", i), + Size: 128, + Ext: ".go", + } + } + benchmarkBuildFileIndex(b, files) +} + +func benchmarkBuildFileIndex(b *testing.B, files []FileInfo) { + b.Helper() + b.ReportAllocs() + b.ResetTimer() + for range b.N { + var err error + benchmarkFileIndex, err = buildFileIndexContext(context.Background(), files, "example.com/bench") + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkTryExactMatchExplicitPath(b *testing.B) { + idx := buildFileIndex(benchmarkFileInventory(50_000), "") + path := "pkg/area-199/feature-499/file-49999.go" + b.ReportAllocs() + b.ResetTimer() + for range b.N { + benchmarkResolved = tryExactMatch(path, idx, "go") + } +} + +func benchmarkScannerTree(b *testing.B, count int) (string, []FileInfo) { + b.Helper() + root := b.TempDir() + if err := os.WriteFile(filepath.Join(root, "go.mod"), []byte("module example.com/bench\n"), 0o644); err != nil { + b.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "shared"), 0o755); err != nil { + b.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "shared", "shared.go"), []byte("package shared\n"), 0o644); err != nil { + b.Fatal(err) + } + files := make([]FileInfo, 0, count) + for i := range count { + dir := filepath.Join(root, "pkg", fmt.Sprintf("area-%03d", i/100)) + if err := os.MkdirAll(dir, 0o755); err != nil { + b.Fatal(err) + } + path := filepath.Join(dir, fmt.Sprintf("file-%05d.go", i)) + if err := os.WriteFile(path, []byte("package bench\n"), 0o644); err != nil { + b.Fatal(err) + } + rel, err := filepath.Rel(root, path) + if err != nil { + b.Fatal(err) + } + files = append(files, FileInfo{Path: rel, Size: 14, Ext: ".go"}) + } + return root, files +} + +func benchmarkFileInventory(count int) []FileInfo { + files := make([]FileInfo, count) + for i := range files { + files[i] = FileInfo{ + Path: fmt.Sprintf("pkg/area-%03d/feature-%03d/file-%05d.go", i%200, i%500, i), + Size: 128, + Ext: ".go", + } + } + return files +} diff --git a/topology/provider_benchmark_test.go b/topology/provider_benchmark_test.go new file mode 100644 index 0000000..609fc94 --- /dev/null +++ b/topology/provider_benchmark_test.go @@ -0,0 +1,90 @@ +package topology + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + "codemap/config" + "codemap/scanner" +) + +var ( + benchmarkManifests []string + benchmarkIdentity CacheIdentity + benchmarkGraph *Graph +) + +func BenchmarkBuildGraphWithProviders(b *testing.B) { + root, _, provider := benchmarkTopologyTree(b, 5_000) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + var err error + benchmarkGraph, benchmarkIdentity, err = BuildGraphWithProviders(context.Background(), root, []Provider{provider}) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkDiscoverManifests(b *testing.B) { + root, _, provider := benchmarkTopologyTree(b, 5_000) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + var err error + benchmarkManifests, err = discoverManifests(context.Background(), root, []Provider{provider}, config.ProjectConfig{}) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkBuildCacheIdentity(b *testing.B) { + root, files, provider := benchmarkTopologyTree(b, 5_000) + manifests, err := discoverManifests(context.Background(), root, []Provider{provider}, config.ProjectConfig{}) + if err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for range b.N { + benchmarkIdentity, err = BuildCacheIdentity(root, files, manifests, []Provider{provider}) + if err != nil { + b.Fatal(err) + } + } +} + +func benchmarkTopologyTree(b *testing.B, count int) (string, []scanner.FileInfo, Provider) { + b.Helper() + root := b.TempDir() + files := make([]scanner.FileInfo, 0, count) + for i := range count { + dir := filepath.Join(root, fmt.Sprintf("module-%03d", i/100)) + if err := os.MkdirAll(dir, 0o755); err != nil { + b.Fatal(err) + } + name := fmt.Sprintf("file-%05d.go", i) + if i%100 == 0 { + name = "bench.module" + } + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte("module benchmark\n"), 0o644); err != nil { + b.Fatal(err) + } + if name == "bench.module" { + continue + } + rel, err := filepath.Rel(root, path) + if err != nil { + b.Fatal(err) + } + files = append(files, scanner.FileInfo{Path: rel, Size: 17, Ext: filepath.Ext(rel)}) + } + provider := stubProvider{name: "benchmark", version: "1", languages: []string{"go"}, manifests: []string{"bench.module"}} + return root, files, provider +} diff --git a/watch/publication_benchmark_test.go b/watch/publication_benchmark_test.go new file mode 100644 index 0000000..bd66b25 --- /dev/null +++ b/watch/publication_benchmark_test.go @@ -0,0 +1,67 @@ +package watch + +import ( + "fmt" + "path/filepath" + "testing" + "time" + + "codemap/config" + "codemap/scanner" +) + +var benchmarkPublishedState State + +func BenchmarkStatePublisherSnapshot(b *testing.B) { + publisher := benchmarkStatePublisher(b, 5_000) + b.ReportAllocs() + b.ResetTimer() + for i := range b.N { + benchmarkPublishedState = publisher.snapshot(uint64(i + 1)) + } +} + +func BenchmarkStatePublisherPublish(b *testing.B) { + publisher := benchmarkStatePublisher(b, 5_000) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + publisher.markDirty(time.Now()) + if err := publisher.publish(); err != nil { + b.Fatal(err) + } + } +} + +func benchmarkStatePublisher(b *testing.B, count int) *statePublisher { + b.Helper() + root := b.TempDir() + files := make(map[string]*scanner.FileInfo, count) + configured := make(map[string]struct{}, count) + imports := make(map[string][]string, count) + importers := make(map[string][]string, count) + for i := range count { + path := fmt.Sprintf("pkg/area-%03d/file-%05d.go", i%100, i) + files[path] = &scanner.FileInfo{Path: path, Size: 128, Ext: ".go"} + configured[path] = struct{}{} + if i > 0 { + previous := fmt.Sprintf("pkg/area-%03d/file-%05d.go", (i-1)%100, i-1) + imports[path] = []string{previous} + importers[previous] = append(importers[previous], path) + } + } + daemon := &Daemon{ + root: root, + graph: &Graph{ + Root: root, + Files: files, + ConfiguredFiles: configured, + FileGraph: &scanner.FileGraph{Root: root, Imports: imports, Importers: importers}, + State: make(map[string]*FileState), + WorkingSet: NewWorkingSet(), + HasDeps: true, + GraphState: newGraphState(root, config.ProjectConfig{}, graphLifecycleAvailable, time.Now(), nil), + }, + } + return newStatePublisher(daemon, filepath.Join(root, "state.json"), "benchmark-instance") +} From 5424a4531f5aba7fa8742626325c7a48d29c8b6d Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:21:34 +0200 Subject: [PATCH 4/7] perf(scanner): Compact dependency graph construction Reuse scan inventories across fallback, CUE, and graph construction. Build compact indexes and initialize only relevant language resolvers while preserving ambiguity checks. --- scanner/cargofallback.go | 14 +- scanner/cue.go | 5 +- scanner/deps_test.go | 6 +- scanner/filegraph.go | 301 ++++++++++++++++++------------ scanner/filegraph_test.go | 84 ++++++++- scanner/hotpath_benchmark_test.go | 22 +++ scanner/importmodel.go | 51 ++++- scanner/importmodel_test.go | 16 ++ scanner/jsworkspace.go | 6 +- scanner/jsworkspace_test.go | 4 +- scanner/outcome.go | 2 + scanner/outcome_test.go | 68 ++++++- scanner/rustaskama_test.go | 9 +- scanner/rustbuildscript.go | 11 +- scanner/rustbuildscript_test.go | 3 - scanner/rustgraph.go | 46 ++--- scanner/types.go | 12 +- scanner/types_test.go | 11 ++ scanner/walker.go | 29 ++- 19 files changed, 487 insertions(+), 213 deletions(-) diff --git a/scanner/cargofallback.go b/scanner/cargofallback.go index 0888ffa..2f77372 100644 --- a/scanner/cargofallback.go +++ b/scanner/cargofallback.go @@ -29,7 +29,11 @@ func buildFileGraphFromOutcomeWithCargoMetadataAndFilters(ctx context.Context, r break } } - fg, err := buildFileGraphFromAnalysesWithCargoMetadataAndFilters(ctx, root, outcome.Analyses, filters, loader, outcome.Sources...) + var inventory []FileInfo + if outcome.hasFileInventory { + inventory = outcome.files + } + fg, err := buildFileGraphFromAnalysesWithCargoMetadataAndFilters(ctx, root, outcome.Analyses, filters, loader, outcome.Sources, inventory, outcome.hasFileInventory) if err != nil { return nil, err } @@ -98,7 +102,9 @@ func scanForGraphOutcomeWithFilters(ctx context.Context, root string, filters Fi return ScanOutcome{}, false, err } fallback := ScanOutcome{ - Sources: []ScanSourceOutcome{incomplete.Outcome}, + Sources: []ScanSourceOutcome{incomplete.Outcome}, + files: files, + hasFileInventory: true, } recovered := false if goFallback, fallbackErr := buildGoFallbackOutcome(ctx, root, files); fallbackErr == nil { @@ -152,6 +158,10 @@ func mergeFallbackOutcome(dst *ScanOutcome, src ScanOutcome) { dst.Analyses = append(dst.Analyses, src.Analyses...) dst.Sources = append(dst.Sources, src.Sources...) dst.precomputedEdges = append(dst.precomputedEdges, src.precomputedEdges...) + if src.hasFileInventory { + dst.files = src.files + dst.hasFileInventory = true + } } func buildCargoFallbackOutcome(ctx context.Context, root string, files []FileInfo, loader cargoMetadataLoader) (ScanOutcome, error) { diff --git a/scanner/cue.go b/scanner/cue.go index d3b1705..c333171 100644 --- a/scanner/cue.go +++ b/scanner/cue.go @@ -19,7 +19,10 @@ func scanCUEFiles(ctx context.Context, root string, filters Filters) (ScanOutcom if err != nil { return ScanOutcome{}, err } - return scanCUEFilesFromFiles(ctx, root, files) + outcome, err := scanCUEFilesFromFiles(ctx, root, files) + outcome.files = files + outcome.hasFileInventory = err == nil + return outcome, err } func scanCUEFilesFromFiles(ctx context.Context, root string, files []FileInfo) (ScanOutcome, error) { diff --git a/scanner/deps_test.go b/scanner/deps_test.go index 0f34e6c..7e1efe2 100644 --- a/scanner/deps_test.go +++ b/scanner/deps_test.go @@ -517,11 +517,11 @@ func TestDepsBuildFileIndex(t *testing.T) { t.Fatalf("expected %q in byDir, got %v", handlerPath, got) } handlerNoExt := strings.TrimSuffix(handlerPath, filepath.Ext(handlerPath)) - if got := idx.byExact[handlerNoExt]; len(got) != 1 || got[0] != handlerPath { - t.Fatalf("expected no-ext exact match for handler.go, got %v", got) + if got := tryExactMatch(handlerNoExt, idx, "go"); len(got) != 1 || got[0] != handlerPath { + t.Fatalf("expected extensionless import to resolve handler.go, got %v", got) } handlerSuffix := filepath.Join("service", "handler.go") - if got := idx.bySuffix[handlerSuffix]; len(got) != 1 || got[0] != handlerPath { + if got := idx.suffixMatches(handlerSuffix); len(got) != 1 || got[0] != handlerPath { t.Fatalf("expected suffix match for service/handler.go, got %v", got) } if got := idx.goPkgs["example.com/project/pkg/service"]; len(got) != 1 || got[0] != handlerPath { diff --git a/scanner/filegraph.go b/scanner/filegraph.go index 90f611a..7a5e07c 100644 --- a/scanner/filegraph.go +++ b/scanner/filegraph.go @@ -27,8 +27,8 @@ type FileGraph struct { // fileIndex provides fast lookup of files by various import-like keys type fileIndex struct { - byExact map[string][]string // exact path -> files - bySuffix map[string][]string // path suffix -> files (for nested packages) + byExact map[string]uint32 // exact path -> inventory count + bySuffix []string // paths ordered by suffix for nested lookup byDir map[string][]string // directory -> files in it goPkgs map[string][]string // Go package path -> files cueModules []cueModuleInfo @@ -61,7 +61,7 @@ func BuildFileGraphFromAnalyses(ctx context.Context, root string, analyses []Fil if err != nil { return nil, err } - return buildFileGraphFromAnalysesWithCargoMetadataAndFilters(ctx, root, filtered, filters, loadCargoMetadata) + return buildFileGraphFromAnalysesWithCargoMetadataAndFilters(ctx, root, filtered, filters, loadCargoMetadata, nil, nil, false) } func buildFileGraphFromAnalysesWithCargoMetadata(ctx context.Context, root string, analyses []FileAnalysis, loader cargoMetadataLoader) (*FileGraph, error) { @@ -71,10 +71,10 @@ func buildFileGraphFromAnalysesWithCargoMetadata(ctx context.Context, root strin if err != nil { return nil, err } - return buildFileGraphFromAnalysesWithCargoMetadataAndFilters(ctx, root, filtered, filters, loader) + return buildFileGraphFromAnalysesWithCargoMetadataAndFilters(ctx, root, filtered, filters, loader, nil, nil, false) } -func buildFileGraphFromAnalysesWithCargoMetadataAndFilters(ctx context.Context, root string, analyses []FileAnalysis, filters Filters, loader cargoMetadataLoader, sources ...ScanSourceOutcome) (*FileGraph, error) { +func buildFileGraphFromAnalysesWithCargoMetadataAndFilters(ctx context.Context, root string, analyses []FileAnalysis, filters Filters, loader cargoMetadataLoader, sources []ScanSourceOutcome, inventory []FileInfo, hasInventory bool) (*FileGraph, error) { if err := ctx.Err(); err != nil { return nil, err } @@ -84,11 +84,10 @@ func buildFileGraphFromAnalysesWithCargoMetadataAndFilters(ctx context.Context, } fg := &FileGraph{ - Root: absRoot, - Imports: make(map[string][]string), - Importers: make(map[string][]string), - Packages: make(map[string][]string), - PathAliases: make(map[string][]string), + Root: absRoot, + Imports: make(map[string][]string), + Importers: make(map[string][]string), + Packages: make(map[string][]string), } hasCargoSource := false for _, source := range sources { @@ -101,19 +100,25 @@ func buildFileGraphFromAnalysesWithCargoMetadataAndFilters(ctx context.Context, // Detect module name from go.mod (for Go import resolution) fg.Module = detectModule(absRoot) - // Detect path aliases from tsconfig.json (for TS/JS import resolution) - fg.PathAliases, fg.BaseURL = detectPathAliases(absRoot) - - useJSWorkspace := needsJSWorkspaceResolver(analyses) - useDartWorkspace := needsDartWorkspaceResolver(analyses) + analysisLanguages := inspectAnalysisLanguages(analyses) + useJSWorkspace := analysisLanguages.hasJS + useDartWorkspace := analysisLanguages.hasDart + if useJSWorkspace { + fg.PathAliases, fg.BaseURL = detectPathAliases(absRoot) + } gitCache := NewGitIgnoreCache(root) scanOnly := filters.Only if useJSWorkspace || useDartWorkspace { scanOnly = nil } - allFiles, err := ScanFiles(ctx, root, gitCache, scanOnly, filters.Exclude) - if err != nil { - return nil, err + var allFiles []FileInfo + if hasInventory && (len(filters.Only) == 0 || !useJSWorkspace && !useDartWorkspace) { + allFiles = inventory + } else { + allFiles, err = ScanFiles(ctx, root, gitCache, scanOnly, filters.Exclude) + if err != nil { + return nil, err + } } files := allFiles if len(filters.Only) > 0 && (useJSWorkspace || useDartWorkspace) { @@ -124,30 +129,30 @@ func buildFileGraphFromAnalysesWithCargoMetadataAndFilters(ctx context.Context, } } } - if !hasCUEAnalyses(analyses) { - for _, file := range files { - if !strings.EqualFold(filepath.Ext(file.Path), ".cue") { - continue - } - cueOutcome, cueErr := scanCUEFilesFromFiles(ctx, absRoot, files) - if cueErr != nil { - return nil, cueErr - } - analyses = append(analyses, cueOutcome.Analyses...) - for _, source := range cueOutcome.Sources { - fg.Coverage.AddSource(source) - } - break + languages := inspectFileLanguages(files) + hasCUEAnalysis := analysisLanguages.hasCUE + if languages.hasCUE && !hasCUEAnalysis { + cueOutcome, cueErr := scanCUEFilesFromFiles(ctx, absRoot, files) + if cueErr != nil { + return nil, cueErr + } + analyses = append(analyses, cueOutcome.Analyses...) + for _, source := range cueOutcome.Sources { + fg.Coverage.AddSource(source) } } - rustWorkspace, cargoOutcome, err := buildRustWorkspaceIndex(ctx, absRoot, analyses, files, loader) - if err != nil { - return nil, err - } - // The outcome already carries cargo provenance; keep exactly one - // cargo-metadata source per graph. - if cargoOutcome != nil && !hasCargoSource { - fg.Coverage.AddSource(*cargoOutcome) + var rustWorkspace *rustWorkspaceIndex + if languages.hasRust || analysisLanguages.hasRust { + var cargoOutcome *ScanSourceOutcome + rustWorkspace, cargoOutcome, err = buildRustWorkspaceIndex(ctx, absRoot, analyses, files, loader) + if err != nil { + return nil, err + } + // The outcome already carries cargo provenance; keep exactly one + // cargo-metadata source per graph. + if cargoOutcome != nil && !hasCargoSource { + fg.Coverage.AddSource(*cargoOutcome) + } } // Build file index for fast fuzzy matching @@ -155,40 +160,39 @@ func buildFileGraphFromAnalysesWithCargoMetadataAndFilters(ctx context.Context, if err != nil { return nil, err } - idx.cueModules = detectCUEModulesWithFiles(absRoot, files) - idx.cuePackages = make(map[string]string) - for _, file := range files { - if DetectLanguage(file.Path) != "cue" { - continue - } - path := filepath.ToSlash(filepath.Clean(file.Path)) - data, readErr := os.ReadFile(filepath.Join(absRoot, filepath.FromSlash(path))) - if readErr == nil { - idx.cuePackages[path], _ = cueHeader(data) + if languages.hasCUE || hasCUEAnalysis { + idx.cueModules = detectCUEModulesWithFiles(absRoot, files) + idx.cuePackages = make(map[string]string) + for _, file := range files { + if fileInfoLanguage(file) != "cue" { + continue + } + path := filepath.ToSlash(filepath.Clean(file.Path)) + data, readErr := os.ReadFile(filepath.Join(absRoot, filepath.FromSlash(path))) + if readErr == nil { + idx.cuePackages[path], _ = cueHeader(data) + } } - } - for _, analysis := range analyses { - if analysis.Language == "cue" && analysis.Package != "" { - idx.cuePackages[filepath.ToSlash(filepath.Clean(analysis.Path))] = analysis.Package + for _, analysis := range analyses { + if analysis.Language == "cue" && analysis.Package != "" { + idx.cuePackages[filepath.ToSlash(filepath.Clean(analysis.Path))] = analysis.Package + } } } fg.Packages = idx.goPkgs - for _, file := range files { + if languages.hasRust { if err := ctx.Err(); err != nil { return nil, err } - if strings.EqualFold(filepath.Ext(file.Path), ".rs") { - if CoverageFromSources(fg.Coverage.Sources).Status != analysis.CoverageUnavailable { - fg.Coverage.AddSource(ScanSourceOutcome{Name: "rust-cargo", Status: ScanSourceMixed, Detail: rustCoverageNote}) - } - break + if CoverageFromSources(fg.Coverage.Sources).Status != analysis.CoverageUnavailable { + fg.Coverage.AddSource(ScanSourceOutcome{Name: "rust-cargo", Status: ScanSourceMixed, Detail: rustCoverageNote}) } } // Languages whose imports name modules rather than files cannot produce // intra-project edges at all, so an empty graph over them is a blind spot // rather than a finding. Recording it here is what keeps --importers and // blast-radius honest too: both read this graph's provenance. - fg.Coverage.AddSymbolLevelImportCoverage(files) + fg.Coverage.addSymbolLevelImportCoverage(languages.symbolLevel) var jsResolver *jsWorkspaceResolver if useJSWorkspace { @@ -283,22 +287,30 @@ func applyPrecomputedFileEdges(fg *FileGraph, edges []fileEdge) { } } -func needsJSWorkspaceResolver(analyses []FileAnalysis) bool { - for _, analysis := range analyses { - if isJavaScriptLanguage(DetectLanguage(analysis.Path)) { - return true - } - } - return false +type analysisLanguageInventory struct { + hasJS bool + hasDart bool + hasRust bool + hasCUE bool } -func needsDartWorkspaceResolver(analyses []FileAnalysis) bool { - for _, file := range analyses { - if DetectLanguage(file.Path) == "dart" { - return true +func inspectAnalysisLanguages(analyses []FileAnalysis) analysisLanguageInventory { + var inventory analysisLanguageInventory + for _, analysis := range analyses { + language := DetectLanguage(analysis.Path) + if isJavaScriptLanguage(language) { + inventory.hasJS = true + } else if language == "dart" { + inventory.hasDart = true + } + switch analysis.Language { + case "rust": + inventory.hasRust = true + case "cue": + inventory.hasCUE = true } } - return false + return inventory } // buildFileIndex creates a multi-key index for fast import resolution @@ -308,12 +320,14 @@ func buildFileIndex(files []FileInfo, goModule string) *fileIndex { } func buildFileIndexContext(ctx context.Context, files []FileInfo, goModule string) (*fileIndex, error) { + directoryHint := min(len(files), 1024) idx := &fileIndex{ - byExact: make(map[string][]string), - bySuffix: make(map[string][]string), - byDir: make(map[string][]string), - goPkgs: make(map[string][]string), + byExact: make(map[string]uint32, len(files)), + bySuffix: make([]string, 0, len(files)), + byDir: make(map[string][]string, directoryHint), + goPkgs: make(map[string][]string, directoryHint), } + goPackagePaths := make(map[string]string, directoryHint) for _, f := range files { if err := ctx.Err(); err != nil { @@ -328,38 +342,30 @@ func buildFileIndexContext(ctx context.Context, files []FileInfo, goModule strin // Index by directory idx.byDir[dir] = append(idx.byDir[dir], path) - // Index by exact path (without extension for fuzzy matching) - idx.byExact[path] = append(idx.byExact[path], path) - noExt := strings.TrimSuffix(path, filepath.Ext(path)) - idx.byExact[noExt] = append(idx.byExact[noExt], path) - - // Index by all path suffixes (for nested package resolution) - // e.g., "llm-server/app/core/config.py" indexed as: - // - "app/core/config.py" - // - "core/config.py" - // - "config.py" - parts := strings.Split(path, string(filepath.Separator)) - for i := 1; i < len(parts); i++ { - if err := ctx.Err(); err != nil { - return nil, err - } - suffix := strings.Join(parts[i:], string(filepath.Separator)) - idx.bySuffix[suffix] = append(idx.bySuffix[suffix], path) - // Also without extension - noExt := strings.TrimSuffix(suffix, filepath.Ext(suffix)) - idx.bySuffix[noExt] = append(idx.bySuffix[noExt], path) - } + idx.byExact[path]++ + idx.bySuffix = append(idx.bySuffix, path) // Go package index. Import paths always use forward slashes, so the // key must be slash-normalized or lookups fail on Windows. if strings.HasSuffix(path, ".go") && !strings.HasSuffix(path, "_test.go") && goModule != "" { pkgPath := goModule if dir != "" { - pkgPath = goModule + "/" + filepath.ToSlash(dir) + var ok bool + pkgPath, ok = goPackagePaths[dir] + if !ok { + pkgPath = goModule + "/" + filepath.ToSlash(dir) + goPackagePaths[dir] = pkgPath + } } idx.goPkgs[pkgPath] = append(idx.goPkgs[pkgPath], path) } } + sort.Slice(idx.bySuffix, func(i, j int) bool { + if order := comparePathsReversed(idx.bySuffix[i], idx.bySuffix[j]); order != 0 { + return order < 0 + } + return idx.bySuffix[i] < idx.bySuffix[j] + }) if err := ctx.Err(); err != nil { return nil, err @@ -367,6 +373,72 @@ func buildFileIndexContext(ctx context.Context, files []FileInfo, goModule strin return idx, nil } +func comparePathsReversed(left, right string) int { + limit := min(len(left), len(right)) + for i := 0; i < limit; i++ { + leftByte, rightByte := left[len(left)-1-i], right[len(right)-1-i] + if leftByte < rightByte { + return -1 + } + if leftByte > rightByte { + return 1 + } + } + if len(left) < len(right) { + return -1 + } + if len(left) > len(right) { + return 1 + } + return 0 +} + +func comparePathSuffix(path, suffix string) int { + prefixLen := len(suffix) + 1 + limit := min(len(path), prefixLen) + for i := 0; i < limit; i++ { + left := path[len(path)-1-i] + right := byte(filepath.Separator) + if i < len(suffix) { + right = suffix[len(suffix)-1-i] + } + if left < right { + return -1 + } + if left > right { + return 1 + } + } + if len(path) < prefixLen { + return -1 + } + if len(path) > prefixLen { + return 1 + } + return 0 +} + +func hasPathSuffix(path, suffix string) bool { + return len(path) > len(suffix) && path[len(path)-len(suffix)-1] == byte(filepath.Separator) && strings.HasSuffix(path, suffix) +} + +func (idx *fileIndex) suffixMatches(suffix string) []string { + if suffix == "" { + return nil + } + start := sort.Search(len(idx.bySuffix), func(i int) bool { + return comparePathSuffix(idx.bySuffix[i], suffix) >= 0 + }) + var matches []string + for i := start; i < len(idx.bySuffix) && hasPathSuffix(idx.bySuffix[i], suffix); i++ { + matches = append(matches, idx.bySuffix[i]) + } + if len(matches) > 1 { + sort.Strings(matches) + } + return matches +} + // fuzzyResolve converts an import path to compatible local file paths. func fuzzyResolve(imp, fromFile string, idx *fileIndex, goModule string, pathAliases map[string][]string, baseURL string) []string { return fuzzyResolveWithWorkspace(imp, fromFile, idx, goModule, pathAliases, baseURL, nil, nil) @@ -474,15 +546,6 @@ func nearestCUEModule(fromFile string, modules []cueModuleInfo) (cueModuleInfo, return cueModuleInfo{}, false } -func hasCUEAnalyses(analyses []FileAnalysis) bool { - for _, analysis := range analyses { - if analysis.Language == "cue" { - return true - } - } - return false -} - func splitCUEImport(imp string) (string, string) { separator := strings.LastIndex(imp, ":") if separator <= strings.LastIndex(imp, "/") { @@ -584,14 +647,13 @@ func resolveRelative(imp, fromDir string, idx *fileIndex, sourceLanguage string) // tryExactMatch looks for exact path matches with common extensions. // Extension list derived from the canonical scanner registry. func tryExactMatch(path string, idx *fileIndex, sourceLanguage string) []string { - extensions := ResolverExtensions() - - for _, ext := range extensions { + if idx.byExact[path] == 1 && languagesCompatible(sourceLanguage, DetectLanguage(path)) { + return []string{path} + } + for _, ext := range resolverExtensions[:len(resolverExtensions)-1] { candidate := path + ext - if files, ok := idx.byExact[candidate]; ok { - if compatible := compatibleFiles(sourceLanguage, files); len(compatible) > 0 { - return compatible - } + if idx.byExact[candidate] == 1 && languagesCompatible(sourceLanguage, DetectLanguage(candidate)) { + return []string{candidate} } } @@ -600,12 +662,9 @@ func tryExactMatch(path string, idx *fileIndex, sourceLanguage string) []string // trySuffixMatch finds files where the path ends with the normalized import func trySuffixMatch(normalized string, idx *fileIndex, sourceLanguage string) []string { - // Extension list derived from the canonical scanner registry. - extensions := ResolverExtensions() - - for _, ext := range extensions { + for _, ext := range resolverExtensions { candidate := normalized + ext - if files, ok := idx.bySuffix[candidate]; ok { + if files := idx.suffixMatches(candidate); len(files) > 0 { files = compatibleFiles(sourceLanguage, files) if len(files) == 0 { continue @@ -621,7 +680,7 @@ func trySuffixMatch(normalized string, idx *fileIndex, sourceLanguage string) [] // Also try __init__.py for Python packages initCandidate := filepath.Join(normalized, "__init__.py") - if files, ok := idx.bySuffix[initCandidate]; ok { + if files := idx.suffixMatches(initCandidate); len(files) > 0 { return compatibleFiles(sourceLanguage, files) } diff --git a/scanner/filegraph_test.go b/scanner/filegraph_test.go index 3052a1b..3524657 100644 --- a/scanner/filegraph_test.go +++ b/scanner/filegraph_test.go @@ -2,14 +2,14 @@ package scanner import ( "context" - - "codemap/analysis" "os" "path/filepath" "reflect" "slices" "sort" "testing" + + "codemap/analysis" ) func TestRustWorkspaceImportersRespectCrateBoundaries(t *testing.T) { @@ -199,8 +199,14 @@ func TestNormalizeImport(t *testing.T) { func TestBuildFileIndex(t *testing.T) { files := []FileInfo{ {Path: "main.go"}, + {Path: "exact.go"}, + {Path: "exact.go.go"}, {Path: filepath.Join("pkg", "util", "helpers.go")}, + {Path: filepath.Join("src", "foobar", "config.py")}, + {Path: filepath.Join("src", "bar", "config.py")}, {Path: filepath.Join("src", "app", "core", "config.py")}, + {Path: filepath.Join("ba", "café", "config.py")}, + {Path: filepath.Join("az", "café", "config.py")}, } idx := buildFileIndex(files, "example.com/project") @@ -212,14 +218,42 @@ func TestBuildFileIndex(t *testing.T) { }{ { name: "exact lookup without extension", - got: idx.byExact[filepath.Join("pkg", "util", "helpers")], + got: tryExactMatch(filepath.Join("pkg", "util", "helpers"), idx, "go"), want: []string{filepath.Join("pkg", "util", "helpers.go")}, }, + { + name: "explicit extension wins before appended extensions", + got: tryExactMatch("exact.go", idx, "go"), + want: []string{"exact.go"}, + }, { name: "suffix lookup for nested path", - got: idx.bySuffix[filepath.Join("app", "core", "config.py")], + got: idx.suffixMatches(filepath.Join("app", "core", "config.py")), want: []string{filepath.Join("src", "app", "core", "config.py")}, }, + { + name: "ambiguous Unicode suffix lookup", + got: idx.suffixMatches(filepath.Join("café", "config.py")), + want: []string{ + filepath.Join("az", "café", "config.py"), + filepath.Join("ba", "café", "config.py"), + }, + }, + { + name: "suffix lookup requires a directory boundary", + got: idx.suffixMatches(filepath.Join("bar", "config.py")), + want: []string{filepath.Join("src", "bar", "config.py")}, + }, + { + name: "exact path is not a nested suffix match", + got: idx.suffixMatches(filepath.Join("src", "bar", "config.py")), + want: nil, + }, + { + name: "empty suffix has no matches", + got: idx.suffixMatches(""), + want: nil, + }, { name: "directory lookup", got: idx.byDir[filepath.Join("pkg", "util")], @@ -239,6 +273,11 @@ func TestBuildFileIndex(t *testing.T) { } }) } + + duplicate := buildFileIndex([]FileInfo{{Path: "duplicate.go"}, {Path: "duplicate.go"}}, "") + if got := tryExactMatch("duplicate", duplicate, "go"); got != nil { + t.Fatalf("duplicate exact match = %#v, want ambiguous", got) + } } func TestResolveRelative(t *testing.T) { @@ -327,6 +366,43 @@ func TestTrySuffixMatch(t *testing.T) { } } +func TestSuffixMatchesMatchesLinearScan(t *testing.T) { + paths := []string{ + filepath.Join("src", "alpha", "config.go"), + filepath.Join("src", "alphabet", "config.go"), + filepath.Join("vendor", "alpha", "config.go"), + filepath.Join("alpha", "nested", "config.go"), + filepath.Join("src", "café", "config.go"), + "config.go", + } + files := make([]FileInfo, len(paths)) + for i, path := range paths { + files[i].Path = path + } + idx := buildFileIndex(files, "") + + for _, suffix := range []string{ + "", + "config.go", + filepath.Join("alpha", "config.go"), + filepath.Join("alphabet", "config.go"), + filepath.Join("nested", "config.go"), + filepath.Join("café", "config.go"), + filepath.Join("missing", "config.go"), + } { + var want []string + for _, path := range paths { + if suffix != "" && hasPathSuffix(path, suffix) { + want = append(want, path) + } + } + sort.Strings(want) + if got := idx.suffixMatches(suffix); !reflect.DeepEqual(got, want) { + t.Fatalf("suffixMatches(%q) = %#v, want %#v", suffix, got, want) + } + } +} + func TestFuzzyResolve(t *testing.T) { files := []FileInfo{ {Path: filepath.Join("pkg", "util", "helpers.go")}, diff --git a/scanner/hotpath_benchmark_test.go b/scanner/hotpath_benchmark_test.go index 9a5925f..c99d44e 100644 --- a/scanner/hotpath_benchmark_test.go +++ b/scanner/hotpath_benchmark_test.go @@ -50,6 +50,28 @@ func BenchmarkBuildFileGraphFromAnalyses(b *testing.B) { } } +func BenchmarkBuildFileGraphFromOutcome(b *testing.B) { + root, files := benchmarkScannerTree(b, 5_000) + analyses := make([]FileAnalysis, 0, len(files)) + for _, file := range files { + analyses = append(analyses, FileAnalysis{ + Path: file.Path, + Language: "go", + Imports: []string{"example.com/bench/shared"}, + }) + } + outcome := ScanOutcome{Analyses: analyses, files: files, hasFileInventory: true} + b.ReportAllocs() + b.ResetTimer() + for range b.N { + var err error + benchmarkGraph, err = BuildFileGraphFromOutcome(context.Background(), root, outcome, Filters{}) + if err != nil { + b.Fatal(err) + } + } +} + func BenchmarkBuildFileIndex(b *testing.B) { files := benchmarkFileInventory(50_000) benchmarkBuildFileIndex(b, files) diff --git a/scanner/importmodel.go b/scanner/importmodel.go index 5e2a8a5..93fcc24 100644 --- a/scanner/importmodel.go +++ b/scanner/importmodel.go @@ -40,17 +40,44 @@ func ResolvesFileLevelImports(language string) bool { return !symbolLevel } -// symbolLevelInventory counts scanned files per symbol-level language, keyed by -// the language's display name. -func symbolLevelInventory(files []FileInfo) map[string]int { - counts := make(map[string]int) +type fileLanguageInventory struct { + hasRust bool + hasCUE bool + symbolLevel map[string]int +} + +func inspectFileLanguages(files []FileInfo) fileLanguageInventory { + var inventory fileLanguageInventory for _, file := range files { - display, symbolLevel := symbolLevelImportLanguages[DetectLanguage(file.Path)] - if symbolLevel { - counts[display]++ + language := fileInfoLanguage(file) + switch language { + case "rust": + inventory.hasRust = true + case "cue": + inventory.hasCUE = true + } + if display, ok := symbolLevelImportLanguages[language]; ok { + if inventory.symbolLevel == nil { + inventory.symbolLevel = make(map[string]int) + } + inventory.symbolLevel[display]++ } } - return counts + return inventory +} + +func fileInfoLanguage(file FileInfo) string { + ext := file.Ext + if ext == "" { + return DetectLanguage(file.Path) + } + return extToLang[strings.ToLower(ext)] +} + +// symbolLevelInventory counts scanned files per symbol-level language, keyed by +// the language's display name. +func symbolLevelInventory(files []FileInfo) map[string]int { + return inspectFileLanguages(files).symbolLevel } // symbolLevelSources renders one source per symbol-level language present, so @@ -104,7 +131,13 @@ func (c *GraphCoverage) AddSymbolLevelImportCoverage(files []FileInfo) { if c == nil { return } - counts := symbolLevelInventory(files) + c.addSymbolLevelImportCoverage(symbolLevelInventory(files)) +} + +func (c *GraphCoverage) addSymbolLevelImportCoverage(counts map[string]int) { + if c == nil { + return + } sources := symbolLevelSources(counts) if len(sources) == 0 { return diff --git a/scanner/importmodel_test.go b/scanner/importmodel_test.go index 9e318e8..fc729cd 100644 --- a/scanner/importmodel_test.go +++ b/scanner/importmodel_test.go @@ -70,6 +70,22 @@ func TestResolvesFileLevelImports(t *testing.T) { } } +func TestInspectFileLanguagesFindsGraphRequirements(t *testing.T) { + inventory := inspectFileLanguages([]FileInfo{ + {Path: "src/lib.rs", Ext: ".rs"}, + {Path: "cue/app.cue", Ext: ".cue"}, + {Path: "src/Main.java", Ext: ".java"}, + {Path: "src/Other.java", Ext: ".java"}, + {Path: "main.go", Ext: ".go"}, + }) + if !inventory.hasRust || !inventory.hasCUE { + t.Fatalf("language requirements = %+v, want Rust and CUE", inventory) + } + if got := inventory.symbolLevel["Java"]; got != 2 { + t.Fatalf("Java file count = %d, want 2", got) + } +} + // Coverage that already knows less must never be talked back up. func TestApplySymbolLevelImportCoverageOnlyRemovesConfidence(t *testing.T) { swift := []FileInfo{{Path: "App/Model.swift"}} diff --git a/scanner/jsworkspace.go b/scanner/jsworkspace.go index 414aeff..f33d3b7 100644 --- a/scanner/jsworkspace.go +++ b/scanner/jsworkspace.go @@ -415,10 +415,8 @@ func resolveManifestTarget(root, target string, idx *fileIndex, sourceLanguage s if !ok { return nil } - for _, candidate := range compatibleFiles(sourceLanguage, idx.byExact[localPath]) { - if candidate == localPath { - return []string{candidate} - } + if idx.byExact[localPath] == 1 && languagesCompatible(sourceLanguage, DetectLanguage(localPath)) { + return []string{localPath} } return nil } diff --git a/scanner/jsworkspace_test.go b/scanner/jsworkspace_test.go index 0d7045e..033352f 100644 --- a/scanner/jsworkspace_test.go +++ b/scanner/jsworkspace_test.go @@ -188,10 +188,10 @@ func TestJSWorkspaceResolverUsesExplicitFilters(t *testing.T) { } func TestJSWorkspaceResolverOnlyWidensJavaScriptScans(t *testing.T) { - if needsJSWorkspaceResolver([]FileAnalysis{{Path: "main.go", Language: "go"}}) { + if inspectAnalysisLanguages([]FileAnalysis{{Path: "main.go", Language: "go"}}).hasJS { t.Fatal("Go-only analysis should not widen the filtered file scan") } - if !needsJSWorkspaceResolver([]FileAnalysis{{Path: "app.ts", Language: "typescript"}}) { + if !inspectAnalysisLanguages([]FileAnalysis{{Path: "app.ts", Language: "typescript"}}).hasJS { t.Fatal("TypeScript analysis should enable workspace manifest scanning") } } diff --git a/scanner/outcome.go b/scanner/outcome.go index 9a75657..3332289 100644 --- a/scanner/outcome.go +++ b/scanner/outcome.go @@ -32,6 +32,8 @@ type ScanOutcome struct { Analyses []FileAnalysis `json:"analyses"` Sources []analysis.Source `json:"sources,omitempty"` precomputedEdges []fileEdge + files []FileInfo + hasFileInventory bool } // GraphCoverage describes graph blind spots and scanner provenance. diff --git a/scanner/outcome_test.go b/scanner/outcome_test.go index 9031dae..29d70b4 100644 --- a/scanner/outcome_test.go +++ b/scanner/outcome_test.go @@ -2,13 +2,13 @@ package scanner import ( "context" - - "codemap/analysis" "errors" "os" "path/filepath" "reflect" "testing" + + "codemap/analysis" ) func requireSourceOutcome(t *testing.T, coverage GraphCoverage, source string) ScanSourceOutcome { @@ -164,6 +164,70 @@ func TestNonCargoGraphHasNoCargoOutcome(t *testing.T) { } } +func TestBuildFileGraphFromOutcomeReusesFileInventory(t *testing.T) { + root := t.TempDir() + outcome := ScanOutcome{ + Analyses: []FileAnalysis{{Path: "app/main.py", Language: "python", Imports: []string{"pkg.util"}}}, + Sources: []ScanSourceOutcome{{Name: "ast-grep", Status: ScanSourceAuthoritative}}, + files: []FileInfo{ + {Path: "app/main.py", Ext: ".py"}, + {Path: filepath.FromSlash("src/pkg/util.py"), Ext: ".py"}, + }, + hasFileInventory: true, + } + + graph, err := BuildFileGraphFromOutcome(context.Background(), root, outcome, Filters{}) + if err != nil { + t.Fatal(err) + } + want := filepath.FromSlash("src/pkg/util.py") + if got := graph.Imports["app/main.py"]; !reflect.DeepEqual(got, []string{want}) { + t.Fatalf("imports = %v, want [%s]", got, want) + } +} + +func TestBuildFileGraphFromOutcomeReusesKnownEmptyInventory(t *testing.T) { + root := filepath.Join(t.TempDir(), "missing") + graph, err := BuildFileGraphFromOutcome(context.Background(), root, ScanOutcome{ + Sources: []ScanSourceOutcome{{Name: "ast-grep", Status: ScanSourceAuthoritative}}, + hasFileInventory: true, + }, Filters{}) + if err != nil { + t.Fatal(err) + } + if len(graph.Imports) != 0 || len(graph.Importers) != 0 { + t.Fatalf("known empty inventory produced edges: %+v", graph) + } +} + +func TestAppendCUEOutcomeReusesFallbackInventory(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "schema.cue"), []byte("package schema\n"), 0o644); err != nil { + t.Fatal(err) + } + outcome, err := appendCUEOutcome(context.Background(), root, Filters{}, ScanOutcome{ + files: []FileInfo{{Path: "schema.cue", Ext: ".cue"}}, + hasFileInventory: true, + }) + if err != nil { + t.Fatal(err) + } + if len(outcome.Analyses) != 1 || outcome.Analyses[0].Path != "schema.cue" || !outcome.hasFileInventory { + t.Fatalf("CUE analyses = %#v", outcome.Analyses) + } +} + +func TestAppendCUEOutcomeReusesKnownEmptyInventory(t *testing.T) { + root := filepath.Join(t.TempDir(), "missing") + outcome, err := appendCUEOutcome(context.Background(), root, Filters{}, ScanOutcome{hasFileInventory: true}) + if err != nil { + t.Fatal(err) + } + if len(outcome.Analyses) != 0 || !outcome.hasFileInventory { + t.Fatalf("outcome = %#v", outcome) + } +} + func TestCoverageFromSourcesMatrix(t *testing.T) { tests := []struct { name string diff --git a/scanner/rustaskama_test.go b/scanner/rustaskama_test.go index a1e6ad1..eec5950 100644 --- a/scanner/rustaskama_test.go +++ b/scanner/rustaskama_test.go @@ -137,9 +137,6 @@ func TestRustAskamaTemplateResolvesWithinCargoPackage(t *testing.T) { } func TestRustAskamaTemplateResolvesWithExtensionSibling(t *testing.T) { - // idx.byExact also indexes files under their extension-stripped key, so - // "app/templates/template.html.orig" appears under the same - // "app/templates/template.html" key as the real target. root := t.TempDir() writeRustCargoFixture(t, root, map[string]string{ "Cargo.toml": "[package]\nname = \"app\"\nversion = \"0.1.0\"\n", @@ -168,14 +165,14 @@ func TestRustAskamaTemplateResolvesWithExtensionSibling(t *testing.T) { } func TestRustAskamaTemplateRequiresAuthoritativeUnambiguousTarget(t *testing.T) { - idx := &fileIndex{byExact: map[string][]string{ - filepath.FromSlash("app/templates/page.html"): {"app/templates/page.html", "app/templates/page.html"}, + idx := &fileIndex{byExact: map[string]uint32{ + filepath.FromSlash("app/templates/page.html"): 2, }} workspace := &rustWorkspaceIndex{packages: []rustPackage{{root: "app", authoritative: true}}} if got := resolveRustAskamaTemplate(t.TempDir(), "app/src/lib.rs", `"page.html"`, idx, workspace); got != "" { t.Fatalf("ambiguous target = %q, want unresolved", got) } - idx.byExact[filepath.FromSlash("app/templates/page.html")] = []string{"app/templates/page.html"} + idx.byExact[filepath.FromSlash("app/templates/page.html")] = 1 workspace.packages[0].authoritative = false if got := resolveRustAskamaTemplate(t.TempDir(), "app/src/lib.rs", `"page.html"`, idx, workspace); got != "" { t.Fatalf("fallback-owned target = %q, want unresolved", got) diff --git a/scanner/rustbuildscript.go b/scanner/rustbuildscript.go index 03b67ec..3a68569 100644 --- a/scanner/rustbuildscript.go +++ b/scanner/rustbuildscript.go @@ -41,15 +41,8 @@ func resolveRustBuildScriptInput(fromFile, input string, idx *fileIndex, workspa if candidate == fromFile { return "" } - // byExact also indexes files under their extension-stripped key, so - // accept only when the target itself is indexed exactly once. - exact := 0 - for _, file := range idx.byExact[candidate] { - if file == candidate { - exact++ - } - } - if exact != 1 { + // Duplicate inventory entries make ownership ambiguous. + if idx.byExact[candidate] != 1 { return "" } return candidate diff --git a/scanner/rustbuildscript_test.go b/scanner/rustbuildscript_test.go index 2ab93d5..f17e2d1 100644 --- a/scanner/rustbuildscript_test.go +++ b/scanner/rustbuildscript_test.go @@ -125,9 +125,6 @@ func TestRustBuildScriptResolvesStaticCargoInputs(t *testing.T) { } func TestRustBuildScriptResolvesTargetWithExtensionSibling(t *testing.T) { - // idx.byExact also indexes files under their extension-stripped key, so - // "app/data.json.gz" appears under the same "app/data.json" key as the - // real target. The real directive must still resolve. root := t.TempDir() writeRustCargoFixture(t, root, map[string]string{ "Cargo.toml": "[package]\nname = \"app\"\nversion = \"0.1.0\"\nbuild = \"build.rs\"\n", diff --git a/scanner/rustgraph.go b/scanner/rustgraph.go index 52561e6..fe422ac 100644 --- a/scanner/rustgraph.go +++ b/scanner/rustgraph.go @@ -510,20 +510,13 @@ func resolveRustExplicitModule(root, declaringFile, literal string) string { } // resolveRustInclude resolves include!(...) relative to the declaring file. -// byExact also indexes files under their extension-stripped key, so accept -// only when the target itself is indexed exactly once. +// Duplicate inventory entries make ownership ambiguous. func resolveRustInclude(root, declaringFile, literal string, idx *fileIndex) string { target := resolveRustExplicitModule(root, declaringFile, literal) if target == "" { return "" } - exact := 0 - for _, file := range idx.byExact[target] { - if file == target { - exact++ - } - } - if exact != 1 { + if idx.byExact[target] != 1 { return "" } return target @@ -534,14 +527,7 @@ func resolveRustEmbeddedFile(root, declaringFile, literal string, idx *fileIndex if target == "" { return "" } - // Extensionless targets can be duplicated by the shared index; require one real path. - exact := 0 - for _, file := range idx.byExact[target] { - if file == target { - exact++ - } - } - if exact != 1 { + if idx.byExact[target] != 1 { return "" } return target @@ -667,15 +653,8 @@ func resolveRustAskamaTemplate(root, fromFile, literal string, idx *fileIndex, w if !pathWithin(target, templateRoot) { return "" } - // byExact also indexes files under their extension-stripped key, so - // accept only when the target itself is indexed exactly once. - exact := 0 - for _, file := range idx.byExact[target] { - if file == target { - exact++ - } - } - if exact != 1 { + // Duplicate inventory entries make ownership ambiguous. + if idx.byExact[target] != 1 { return "" } return target @@ -744,8 +723,8 @@ func resolveRustModule(name, fromFile string, idx *fileIndex, workspace *rustWor filepath.Join(dir, name+".rs"), filepath.Join(dir, name, "mod.rs"), } { - if files := idx.byExact[candidate]; len(files) == 1 { - return files[0] + if idx.byExact[candidate] == 1 { + return candidate } } return "" @@ -912,8 +891,8 @@ func resolveRustPathFromDirectory(base string, parts []string, idx *fileIndex) s for i := len(parts); i > 0; i-- { modulePath := filepath.Join(append([]string{base}, parts[:i]...)...) for _, candidate := range []string{modulePath + ".rs", filepath.Join(modulePath, "mod.rs")} { - if files := idx.byExact[candidate]; len(files) == 1 { - return files[0] + if idx.byExact[candidate] == 1 { + return candidate } } } @@ -996,8 +975,7 @@ func (index *rustWorkspaceIndex) targetForFile(path string, idx *fileIndex) (rus } func rustTargetIndexed(target rustTarget, idx *fileIndex) bool { - files := idx.byExact[target.rootFile] - return len(files) == 1 && files[0] == target.rootFile + return idx.byExact[target.rootFile] == 1 } func pathWithin(path, dir string) bool { @@ -1044,11 +1022,11 @@ func (index *rustWorkspaceIndex) targetContainsFile(target rustTarget, path stri modulePath := filepath.Join(append([]string{target.sourceDir}, parts[:i+1]...)...) var next string for _, candidate := range []string{modulePath + ".rs", filepath.Join(modulePath, "mod.rs")} { - if files := idx.byExact[candidate]; len(files) == 1 { + if idx.byExact[candidate] == 1 { if next != "" { return false } - next = files[0] + next = candidate } } if next == "" { diff --git a/scanner/types.go b/scanner/types.go index 42b9190..47b0ddd 100644 --- a/scanner/types.go +++ b/scanner/types.go @@ -258,11 +258,15 @@ func PromptExtensions() []string { return exts } -// ResolverExtensions returns extensions used for import path resolution, -// including index-file patterns for JS/TS/Python ecosystems. -// Sorted by length descending so longer extensions match first (.tsx before .ts), -// with empty string last as the final fallback. +var resolverExtensions = buildResolverExtensions() + +// ResolverExtensions returns an isolated copy of the ordered extensions used +// for import path resolution. func ResolverExtensions() []string { + return slices.Clone(resolverExtensions) +} + +func buildResolverExtensions() []string { var exts []string for ext := range extToLang { exts = append(exts, ext) diff --git a/scanner/types_test.go b/scanner/types_test.go index 0a85feb..9164247 100644 --- a/scanner/types_test.go +++ b/scanner/types_test.go @@ -15,3 +15,14 @@ func TestNewDepsProjectWithCoverageAndFiltersClonesEffectiveFilters(t *testing.T t.Fatalf("effective filters were not cloned: %+v", project.EffectiveFilters) } } + +func TestResolverExtensionsReturnsIsolatedOrder(t *testing.T) { + first := ResolverExtensions() + if len(first) == 0 || first[len(first)-1] != "" { + t.Fatalf("extensions = %#v, want bare-path fallback last", first) + } + first[0] = "changed" + if second := ResolverExtensions(); second[0] == "changed" { + t.Fatal("caller mutation changed resolver extension order") + } +} diff --git a/scanner/walker.go b/scanner/walker.go index 0b17c0a..40042a3 100644 --- a/scanner/walker.go +++ b/scanner/walker.go @@ -262,19 +262,16 @@ func ScanFiles(ctx context.Context, root string, cache *GitIgnoreCache, only []s return nil } - // Compute absolute path once for gitignore checks and relative path calculation - absPath, _ := filepath.Abs(path) - // For directories: load any .gitignore, then check if dir itself should be skipped if info.IsDir() { if cache != nil { - cache.tryLoadGitignore(absPath) - if cache.ShouldIgnore(absPath) { + cache.tryLoadGitignore(path) + if cache.ShouldIgnore(path) { return filepath.SkipDir } } // Check if directory matches any exclude pattern - relPath, _ := filepath.Rel(absRoot, absPath) + relPath, _ := filepath.Rel(absRoot, path) if relPath != "." { for _, pattern := range exclude { pattern = strings.TrimSpace(pattern) @@ -287,11 +284,11 @@ func ScanFiles(ctx context.Context, root string, cache *GitIgnoreCache, only []s } // For files: check gitignore - if cache != nil && cache.ShouldIgnore(absPath) { + if cache != nil && cache.ShouldIgnore(path) { return nil } - relPath, _ := filepath.Rel(absRoot, absPath) + relPath, _ := filepath.Rel(absRoot, path) ext := filepath.Ext(path) // Apply user filters (--only and --exclude) @@ -382,12 +379,26 @@ func ScanForDeps(ctx context.Context, root string, filters Filters) (ScanOutcome if err != nil { return outcome, err } - cueOutcome, err := scanCUEFiles(ctx, root, filters) + return appendCUEOutcome(ctx, root, filters, outcome) +} + +func appendCUEOutcome(ctx context.Context, root string, filters Filters, outcome ScanOutcome) (ScanOutcome, error) { + var cueOutcome ScanOutcome + var err error + if outcome.hasFileInventory { + cueOutcome, err = scanCUEFilesFromFiles(ctx, root, outcome.files) + } else { + cueOutcome, err = scanCUEFiles(ctx, root, filters) + } if err != nil { return ScanOutcome{}, err } outcome.Analyses = append(outcome.Analyses, cueOutcome.Analyses...) outcome.Sources = append(outcome.Sources, cueOutcome.Sources...) + if cueOutcome.hasFileInventory { + outcome.files = cueOutcome.files + outcome.hasFileInventory = true + } return outcome, nil } From 2c44955431b1b6753b7433fb6cd85bbfa245cec8 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:21:42 +0200 Subject: [PATCH 5/7] perf(topology): Share manifest and source discovery Collect provider files and manifests in one filtered walk. Hash cache inputs directly to avoid formatting and repeated path normalization. --- topology/cache.go | 69 +++++++++++++++++--- topology/cache_test.go | 17 +++++ topology/graph.go | 19 +++--- topology/graph_test.go | 12 ++++ topology/graph_windows_test.go | 11 ++++ topology/provider.go | 116 +++++++++++++++++++-------------- topology/provider_test.go | 60 +++++++++++++++++ 7 files changed, 237 insertions(+), 67 deletions(-) create mode 100644 topology/graph_windows_test.go diff --git a/topology/cache.go b/topology/cache.go index 0435f3b..d42394b 100644 --- a/topology/cache.go +++ b/topology/cache.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "time" @@ -42,6 +43,10 @@ func CachePathAt(cacheDir string) string { } func BuildCacheIdentity(root string, files []scanner.FileInfo, manifests []string, providers []Provider) (CacheIdentity, error) { + absRoot, err := filepath.Abs(root) + if err != nil { + return CacheIdentity{}, err + } cfg := config.Load(root) filterData, err := json.Marshal(cfg) if err != nil { @@ -52,28 +57,67 @@ func BuildCacheIdentity(root string, files []scanner.FileInfo, manifests []strin sort.Strings(manifestPaths) manifestHash := sha256.New() for _, manifest := range manifestPaths { - rel, err := normalizeRepoPath(root, manifest) + rel, err := normalizeRepoPathFromRoot(absRoot, manifest) if err != nil { return CacheIdentity{}, fmt.Errorf("manifest %q: %w", manifest, err) } - data, err := os.ReadFile(filepath.Join(root, rel)) + data, err := os.ReadFile(filepath.Join(absRoot, rel)) if err != nil { return CacheIdentity{}, err } writeHashPart(manifestHash, filepath.ToSlash(rel)) - writeHashPart(manifestHash, string(data)) + writeHashBytes(manifestHash, data) } - fileParts := make([]string, 0, len(files)) - for _, file := range files { - rel, err := normalizeRepoPath(root, file.Path) + type fileIdentity struct { + path string + index int + } + fileParts := make([]fileIdentity, 0, len(files)) + for index, file := range files { + rel, err := normalizeRepoPathFromRoot(absRoot, file.Path) if err != nil { return CacheIdentity{}, fmt.Errorf("configured file %q: %w", file.Path, err) } - fileParts = append(fileParts, fmt.Sprintf("%s\x00%d\x00%s\x00%t\x00%d\x00%d", - filepath.ToSlash(rel), file.Size, file.Ext, file.IsNew, file.Added, file.Removed)) + fileParts = append(fileParts, fileIdentity{path: filepath.ToSlash(rel), index: index}) + } + sort.Slice(fileParts, func(i, j int) bool { + if fileParts[i].path != fileParts[j].path { + return fileParts[i].path < fileParts[j].path + } + left, right := files[fileParts[i].index], files[fileParts[j].index] + if left.Size != right.Size { + return left.Size < right.Size + } + if left.Ext != right.Ext { + return left.Ext < right.Ext + } + if left.IsNew != right.IsNew { + return !left.IsNew + } + if left.Added != right.Added { + return left.Added < right.Added + } + return left.Removed < right.Removed + }) + fileHash := sha256.New() + record := make([]byte, 0, 128) + for _, part := range fileParts { + file := files[part.index] + record = record[:0] + record = append(record, part.path...) + record = append(record, 0) + record = strconv.AppendInt(record, file.Size, 10) + record = append(record, 0) + record = append(record, file.Ext...) + record = append(record, 0) + record = strconv.AppendBool(record, file.IsNew) + record = append(record, 0) + record = strconv.AppendInt(record, int64(file.Added), 10) + record = append(record, 0) + record = strconv.AppendInt(record, int64(file.Removed), 10) + writeHashBytes(fileHash, record) } - sort.Strings(fileParts) providerParts := make([]string, 0, len(providers)) for _, provider := range sortedProviders(providers) { @@ -83,7 +127,7 @@ func BuildCacheIdentity(root string, files []scanner.FileInfo, manifests []strin return CacheIdentity{ Filters: hashStrings(string(filterData)), Manifests: hex.EncodeToString(manifestHash.Sum(nil)), - ConfiguredFiles: hashStrings(fileParts...), + ConfiguredFiles: hex.EncodeToString(fileHash.Sum(nil)), ProviderVersions: hashStrings(providerParts...), }, nil } @@ -222,3 +266,8 @@ func writeHashPart(hash interface{ Write([]byte) (int, error) }, part string) { _, _ = hash.Write([]byte(part)) _, _ = hash.Write([]byte{0}) } + +func writeHashBytes(hash interface{ Write([]byte) (int, error) }, part []byte) { + _, _ = hash.Write(part) + _, _ = hash.Write([]byte{0}) +} diff --git a/topology/cache_test.go b/topology/cache_test.go index 2d7821c..6018b9d 100644 --- a/topology/cache_test.go +++ b/topology/cache_test.go @@ -156,6 +156,23 @@ func TestCacheIdentityChangesForFiltersManifestsFilesAndProviderVersions(t *test if err != nil { t.Fatal(err) } + reordered, err := BuildCacheIdentity(root, []scanner.FileInfo{ + {Path: "other.go", Size: 42, Ext: ".go", IsNew: true, Added: 3, Removed: 1}, + {Path: "main.go"}, + }, []string{"go.mod"}, providers) + if err != nil { + t.Fatal(err) + } + reorderedAgain, err := BuildCacheIdentity(root, []scanner.FileInfo{ + {Path: "main.go"}, + {Path: "other.go", Size: 42, Ext: ".go", IsNew: true, Added: 3, Removed: 1}, + }, []string{"go.mod"}, providers) + if err != nil { + t.Fatal(err) + } + if reordered.ConfiguredFiles != reorderedAgain.ConfiguredFiles { + t.Fatal("configured file identity depends on inventory order") + } writeTopologyFixture(t, root, ".codemap/config.json", `{"only":["go"],"exclude":["vendor"]}`) filtered, err := BuildCacheIdentity(root, files, []string{"go.mod"}, providers) diff --git a/topology/graph.go b/topology/graph.go index 1f165d8..7320f7b 100644 --- a/topology/graph.go +++ b/topology/graph.go @@ -303,21 +303,22 @@ func validateNodePath(root string, node Node) error { } func normalizeRepoPath(root, path string) (string, error) { - if path == "" || filepath.IsAbs(path) { + absRoot, err := filepath.Abs(root) + if err != nil { + return "", err + } + return normalizeRepoPathFromRoot(absRoot, path) +} + +func normalizeRepoPathFromRoot(absRoot, path string) (string, error) { + if path == "" || filepath.IsAbs(path) || filepath.VolumeName(path) != "" { return "", fmt.Errorf("path must be non-empty and repository-relative") } clean := filepath.Clean(path) if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { return "", fmt.Errorf("path escapes repository") } - absRoot, err := filepath.Abs(root) - if err != nil { - return "", err - } - joined, err := filepath.Abs(filepath.Join(absRoot, clean)) - if err != nil { - return "", err - } + joined := filepath.Join(absRoot, clean) rel, err := filepath.Rel(absRoot, joined) if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { return "", fmt.Errorf("path escapes repository") diff --git a/topology/graph_test.go b/topology/graph_test.go index 5d6dc6f..3b4b5e3 100644 --- a/topology/graph_test.go +++ b/topology/graph_test.go @@ -106,6 +106,18 @@ func TestMergeFragmentsRejectsEscapingPathsAndUnknownEndpoints(t *testing.T) { } } +func TestNormalizeRepoPathRejectsAbsoluteAndEscapingPaths(t *testing.T) { + root := t.TempDir() + for _, path := range []string{"", filepath.Join("..", "outside"), filepath.Join(root, "inside")} { + if _, err := normalizeRepoPath(root, path); err == nil { + t.Fatalf("normalizeRepoPath(%q) succeeded", path) + } + } + if got, err := normalizeRepoPath(root, filepath.Join("nested", "file.go")); err != nil || got != filepath.Join("nested", "file.go") { + t.Fatalf("normalizeRepoPath(valid) = %q, %v", got, err) + } +} + func TestMergeFragmentsRejectsMissingNodePaths(t *testing.T) { root := t.TempDir() missing := testNode("test:missing", "missing") diff --git a/topology/graph_windows_test.go b/topology/graph_windows_test.go new file mode 100644 index 0000000..e0c42a2 --- /dev/null +++ b/topology/graph_windows_test.go @@ -0,0 +1,11 @@ +//go:build windows + +package topology + +import "testing" + +func TestNormalizeRepoPathRejectsVolumePaths(t *testing.T) { + if _, err := normalizeRepoPath(`C:\repo`, `D:file.go`); err == nil { + t.Fatal("drive-relative path succeeded") + } +} diff --git a/topology/provider.go b/topology/provider.go index 64ecbbb..7ebd190 100644 --- a/topology/provider.go +++ b/topology/provider.go @@ -17,6 +17,8 @@ import ( const maxManifestWalkEntries = 100_000 +var errManifestWalkLimit = errors.New("manifest walk limit exceeded") + type ManifestSelector struct { Names []string } @@ -109,25 +111,20 @@ func BuildGraphWithProviders(ctx context.Context, root string, providers []Provi return MergeFragments(root, nil), CacheIdentity{}, nil } - cache := scanner.NewGitIgnoreCache(root) - inventoryFiles, err := scanner.ScanConfiguredFiles(ctx, root, cache) + inventoryFiles, manifests, err := discoverInventory(ctx, root, selected, cfg, true) if err != nil { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return nil, CacheIdentity{}, err } - return unavailableGraph("inventory-failed", err.Error()), CacheIdentity{}, nil + code := "inventory-failed" + if errors.Is(err, errManifestWalkLimit) { + code = "manifest-discovery-failed" + } + return unavailableGraph(code, err.Error()), CacheIdentity{}, nil } if err := ctx.Err(); err != nil { return nil, CacheIdentity{}, err } - inventoryFiles = filterInventoryFiles(inventoryFiles, selected) - manifests, err := discoverManifests(ctx, root, selected, cfg) - if err != nil { - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return nil, CacheIdentity{}, err - } - return unavailableGraph("manifest-discovery-failed", err.Error()), CacheIdentity{}, nil - } identity, err := BuildCacheIdentity(root, inventoryFiles, manifests, selected) if err != nil { return unavailableGraph("cache-identity-failed", err.Error()), CacheIdentity{}, nil @@ -181,10 +178,20 @@ func BuildGraphWithProviders(ctx context.Context, root string, providers []Provi } func discoverManifests(ctx context.Context, root string, providers []Provider, cfg config.ProjectConfig) ([]string, error) { + _, manifests, err := discoverInventory(ctx, root, providers, cfg, false) + return manifests, err +} + +func discoverInventory(ctx context.Context, root string, providers []Provider, cfg config.ProjectConfig, includeFiles bool) ([]scanner.FileInfo, []string, error) { + return discoverInventoryWithLimit(ctx, root, providers, cfg, includeFiles, maxManifestWalkEntries) +} + +func discoverInventoryWithLimit(ctx context.Context, root string, providers []Provider, cfg config.ProjectConfig, includeFiles bool, maxEntries int) ([]scanner.FileInfo, []string, error) { if err := ctx.Err(); err != nil { - return nil, err + return nil, nil, err } names := make(map[string]bool) + languages := make(map[string]bool) for _, provider := range providers { for _, name := range provider.Manifests().Names { name = strings.TrimSpace(name) @@ -192,32 +199,44 @@ func discoverManifests(ctx context.Context, root string, providers []Provider, c names[name] = true } } + if includeFiles { + for _, language := range provider.Languages() { + language = strings.TrimPrefix(strings.ToLower(strings.TrimSpace(language)), ".") + if language != "" { + languages[language] = true + } + } + } } - if len(names) == 0 { - return nil, nil + if len(names) == 0 && !includeFiles { + return nil, nil, nil } - absRoot, err := filepath.Abs(root) - if err != nil { - return nil, err - } + absRoot := projectpath.CanonicalPath(root) ignoreCache := scanner.NewGitIgnoreCache(absRoot) entries := 0 + var files []scanner.FileInfo var manifests []string - err = filepath.WalkDir(absRoot, func(path string, entry fs.DirEntry, walkErr error) error { + var sourceIgnoredRoot string + err := filepath.WalkDir(absRoot, func(path string, entry fs.DirEntry, walkErr error) error { if err := ctx.Err(); err != nil { return err } if walkErr != nil { return walkErr } - entries++ - if entries > maxManifestWalkEntries { - return fmt.Errorf("manifest walk exceeded %d entries", maxManifestWalkEntries) + if len(names) > 0 { + entries++ + if entries > maxEntries { + return fmt.Errorf("%w: exceeded %d entries", errManifestWalkLimit, maxEntries) + } } if path == absRoot { return nil } + if sourceIgnoredRoot != "" && path != sourceIgnoredRoot && !strings.HasPrefix(path, sourceIgnoredRoot+string(filepath.Separator)) { + sourceIgnoredRoot = "" + } rel, err := filepath.Rel(absRoot, path) if err != nil { return err @@ -228,20 +247,42 @@ func discoverManifests(ctx context.Context, root string, providers []Provider, c return filepath.SkipDir } ignoreCache.EnsureDir(path) + if sourceIgnoredRoot == "" && scanner.IgnoredDirs[entry.Name()] { + sourceIgnoredRoot = path + } return nil } - if !names[entry.Name()] || ignoreCache.ShouldIgnore(path) || - !scanner.MatchesFilters(filepath.ToSlash(rel), filepath.Ext(rel), nil, cfg.Exclude) { + ext := filepath.Ext(rel) + relSlash := filepath.ToSlash(rel) + if ignoreCache.ShouldIgnore(path) || !scanner.MatchesFilters(relSlash, ext, nil, cfg.Exclude) { return nil } - manifests = append(manifests, filepath.Clean(rel)) + if names[entry.Name()] { + manifests = append(manifests, filepath.Clean(rel)) + } + if !includeFiles || sourceIgnoredRoot != "" || scanner.IgnoredDirs[entry.Name()] || + !scanner.MatchesFilters(relSlash, ext, cfg.Only, nil) { + return nil + } + language := strings.ToLower(scanner.DetectLanguage(relSlash)) + if !languages[language] && !languages[strings.TrimPrefix(strings.ToLower(ext), ".")] { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + files = append(files, scanner.FileInfo{Path: filepath.Clean(rel), Size: info.Size(), Ext: ext}) return nil }) if err != nil { - return nil, err + return nil, nil, err } sort.Strings(manifests) - return manifests, nil + if err := ctx.Err(); err != nil { + return nil, nil, err + } + return files, manifests, nil } func enabledProviders(providers []Provider, only []string) []Provider { @@ -268,27 +309,6 @@ func enabledProviders(providers []Provider, only []string) []Provider { return enabled } -func filterInventoryFiles(files []scanner.FileInfo, providers []Provider) []scanner.FileInfo { - languages := make(map[string]bool) - for _, provider := range providers { - for _, language := range provider.Languages() { - language = strings.TrimPrefix(strings.ToLower(strings.TrimSpace(language)), ".") - if language != "" { - languages[language] = true - } - } - } - filtered := make([]scanner.FileInfo, 0, len(files)) - for _, file := range files { - language := strings.ToLower(scanner.DetectLanguage(file.Path)) - extension := strings.TrimPrefix(strings.ToLower(file.Ext), ".") - if languages[language] || languages[extension] { - filtered = append(filtered, file) - } - } - return filtered -} - func sortedProviders(providers []Provider) []Provider { result := append([]Provider(nil), providers...) sort.Slice(result, func(i, j int) bool { diff --git a/topology/provider_test.go b/topology/provider_test.go index 0afd260..cfdc6b2 100644 --- a/topology/provider_test.go +++ b/topology/provider_test.go @@ -259,6 +259,54 @@ func TestDiscoverManifestsHonorsGitignoreAndExclude(t *testing.T) { } } +func TestDiscoverInventoryFindsManifestsBelowSourceIgnoredDirectories(t *testing.T) { + root := t.TempDir() + writeTopologyFixture(t, root, "Main.java", "class Main {}\n") + writeTopologyFixture(t, root, "main.go", "package main\n") + writeTopologyFixture(t, root, "pom.xml", "") + writeTopologyFixture(t, root, "vendor/pom.xml", "") + writeTopologyFixture(t, root, "vendor/generated.go", "package generated\n") + writeTopologyFixture(t, root, "vendor/.gitignore", "ignored/\n") + writeTopologyFixture(t, root, "vendor/ignored/pom.xml", "") + writeTopologyFixture(t, root, "build/pom.xml", "") + writeTopologyFixture(t, root, "build/generated.go", "package generated\n") + + files, manifests, err := discoverInventory(context.Background(), root, []Provider{stubProvider{ + name: "jvm", + version: "1", + languages: []string{"java"}, + manifests: []string{"pom.xml"}, + }}, config.ProjectConfig{}, true) + if err != nil { + t.Fatal(err) + } + wantFiles := []string{"Main.java"} + gotFiles := make([]string, len(files)) + for i, file := range files { + gotFiles[i] = filepath.ToSlash(file.Path) + } + if !reflect.DeepEqual(gotFiles, wantFiles) { + t.Fatalf("files = %#v, want %#v", gotFiles, wantFiles) + } + wantManifests := []string{"build/pom.xml", "pom.xml", "vendor/pom.xml"} + for i := range manifests { + manifests[i] = filepath.ToSlash(manifests[i]) + } + if !reflect.DeepEqual(manifests, wantManifests) { + t.Fatalf("manifests = %#v, want %#v", manifests, wantManifests) + } + + filteredFiles, filteredManifests, err := discoverInventory(context.Background(), root, []Provider{stubProvider{ + name: "jvm", version: "1", languages: []string{"java"}, manifests: []string{"pom.xml"}, + }}, config.ProjectConfig{Only: []string{"java"}}, true) + if err != nil { + t.Fatal(err) + } + if len(filteredFiles) != 1 || filepath.ToSlash(filteredFiles[0].Path) != "Main.java" || !reflect.DeepEqual(filteredManifests, manifests) { + t.Fatalf("filtered inventory = (%#v, %#v), want the Java file and unchanged manifests", filteredFiles, filteredManifests) + } +} + func TestDiscoverManifestsStopsOnCancellation(t *testing.T) { root := t.TempDir() writeTopologyFixture(t, root, "pom.xml", "") @@ -276,6 +324,18 @@ func TestDiscoverManifestsStopsOnCancellation(t *testing.T) { } } +func TestDiscoverInventoryReportsManifestWalkLimit(t *testing.T) { + root := t.TempDir() + writeTopologyFixture(t, root, "pom.xml", "") + + _, _, err := discoverInventoryWithLimit(context.Background(), root, []Provider{stubProvider{ + name: "jvm", version: "1", languages: []string{"java"}, manifests: []string{"pom.xml"}, + }}, config.ProjectConfig{}, false, 1) + if !errors.Is(err, errManifestWalkLimit) { + t.Fatalf("error = %v, want manifest walk limit", err) + } +} + func TestRegisteredProvidersAreNameSorted(t *testing.T) { providerRegistryMu.Lock() original := append([]Provider(nil), providerRegistry...) From 7553fc33faf412bf92e87d1eec4c8d93bc15f1d3 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:21:44 +0200 Subject: [PATCH 6/7] perf(render): Bound large-inventory aggregation Cache tree statistics and retain only the largest files. Compute skyline totals without copying the full source inventory. --- render/skyline.go | 94 +++++++++++++++++++++++++----------------- render/skyline_test.go | 68 ++++++++++++++---------------- render/tree.go | 94 +++++++++++++++++++++++++++--------------- render/tree_test.go | 76 +++++++++++++++------------------- 4 files changed, 179 insertions(+), 153 deletions(-) diff --git a/render/skyline.go b/render/skyline.go index b21b011..8344b32 100644 --- a/render/skyline.go +++ b/render/skyline.go @@ -78,45 +78,70 @@ type extAgg struct { count int } -// filterCodeFiles returns only source code files -func filterCodeFiles(files []scanner.FileInfo) []scanner.FileInfo { - var result []scanner.FileInfo - for _, f := range files { - if codeExtensions[strings.ToLower(f.Ext)] || codeFilenames[filepath.Base(f.Path)] { - result = append(result, f) - } - } - if len(result) == 0 { - return files - } - return result -} - // aggregateByExtension groups files by extension func aggregateByExtension(files []scanner.FileInfo) []extAgg { - groups := make(map[string]*extAgg) + groups := make(map[string]extAgg) for _, f := range files { ext := strings.ToLower(f.Ext) if ext == "" { ext = filepath.Base(f.Path) } - if groups[ext] == nil { - groups[ext] = &extAgg{ext: ext} - } - groups[ext].size += f.Size - groups[ext].count++ + agg := groups[ext] + agg.ext = ext + agg.size += f.Size + agg.count++ + groups[ext] = agg } var result []extAgg for _, agg := range groups { - result = append(result, *agg) + result = append(result, agg) } - sort.Slice(result, func(i, j int) bool { - return result[i].size > result[j].size - }) + sortExtensionAggregates(result) return result } +func aggregateSkylineFiles(files []scanner.FileInfo) ([]extAgg, int, int64) { + groups := make(map[string]extAgg) + count := 0 + var size, allSize int64 + for _, file := range files { + allSize += file.Size + ext := strings.ToLower(file.Ext) + if !codeExtensions[ext] && !codeFilenames[filepath.Base(file.Path)] { + continue + } + if ext == "" { + ext = filepath.Base(file.Path) + } + agg := groups[ext] + agg.ext = ext + agg.size += file.Size + agg.count++ + groups[ext] = agg + count++ + size += file.Size + } + if count == 0 { + return aggregateByExtension(files), len(files), allSize + } + result := make([]extAgg, 0, len(groups)) + for _, agg := range groups { + result = append(result, agg) + } + sortExtensionAggregates(result) + return result, count, size +} + +func sortExtensionAggregates(aggregates []extAgg) { + sort.Slice(aggregates, func(i, j int) bool { + if aggregates[i].size != aggregates[j].size { + return aggregates[i].size > aggregates[j].size + } + return aggregates[i].ext < aggregates[j].ext + }) +} + // getBuildingChar returns building texture character func getBuildingChar(ext string) rune { ext = strings.ToLower(ext) @@ -220,8 +245,7 @@ func Skyline(w io.Writer, project scanner.Project, animate bool) { width = 80 } - codeFiles := filterCodeFiles(files) - sorted := aggregateByExtension(codeFiles) + sorted, codeFileCount, codeSize := aggregateSkylineFiles(files) arranged := createBuildings(sorted, width) if len(arranged) == 0 { @@ -242,15 +266,15 @@ func Skyline(w io.Writer, project scanner.Project, animate bool) { // If writer is not os.Stdout, disable animation if animate && w == os.Stdout { - renderAnimated(w, arranged, width, leftMargin, sceneLeft, sceneRight, sceneWidth, codeFiles, projectName, sorted) + renderAnimated(w, arranged, width, leftMargin, sceneLeft, sceneRight, sceneWidth, codeFileCount, codeSize, projectName, sorted) } else { - renderStatic(w, arranged, width, leftMargin, sceneLeft, sceneRight, sceneWidth, codeFiles, projectName, sorted) + renderStatic(w, arranged, width, leftMargin, sceneLeft, sceneRight, sceneWidth, codeFileCount, codeSize, projectName, sorted) } } // renderStatic renders static skyline to the given writer func renderStatic(w io.Writer, arranged []building, width, leftMargin, sceneLeft, sceneRight, sceneWidth int, - codeFiles []scanner.FileInfo, projectName string, sorted []extAgg) { + codeFileCount int, codeSize int64, projectName string, sorted []extAgg) { // Build grid grid := make([][]rune, skyHeight+maxHeight+1) for i := range grid { @@ -383,11 +407,7 @@ func renderStatic(w io.Writer, arranged []building, width, leftMargin, sceneLeft title := fmt.Sprintf("─── %s ───", projectName) fmt.Fprintf(w, "%s%s%s\n", BoldWhite, CenterString(title, width), Reset) - var codeSize int64 - for _, f := range codeFiles { - codeSize += f.Size - } - stats := fmt.Sprintf("%d languages · %d files · %s", len(sorted), len(codeFiles), formatSize(codeSize)) + stats := fmt.Sprintf("%d languages · %d files · %s", len(sorted), codeFileCount, formatSize(codeSize)) fmt.Fprintf(w, "%s%s%s\n", Cyan, CenterString(stats, width), Reset) fmt.Fprintln(w) } @@ -400,7 +420,6 @@ type animationModel struct { sceneLeft int sceneRight int sceneWidth int - codeFiles []scanner.FileInfo projectName string sorted []extAgg starPositions [][2]int @@ -602,7 +621,7 @@ func (m animationModel) View() string { // renderAnimated renders animated skyline using bubbletea func renderAnimated(w io.Writer, arranged []building, width, leftMargin, sceneLeft, sceneRight, sceneWidth int, - codeFiles []scanner.FileInfo, projectName string, sorted []extAgg) { + codeFileCount int, codeSize int64, projectName string, sorted []extAgg) { // Generate star positions var starPositions [][2]int for row := 0; row < skyHeight; row++ { @@ -629,7 +648,6 @@ func renderAnimated(w io.Writer, arranged []building, width, leftMargin, sceneLe sceneLeft: sceneLeft, sceneRight: sceneRight, sceneWidth: sceneWidth, - codeFiles: codeFiles, projectName: projectName, sorted: sorted, starPositions: starPositions, @@ -643,7 +661,7 @@ func renderAnimated(w io.Writer, arranged []building, width, leftMargin, sceneLe p.Run() // After animation, print static final frame to main screen - renderStatic(w, arranged, width, leftMargin, sceneLeft, sceneRight, sceneWidth, codeFiles, projectName, sorted) + renderStatic(w, arranged, width, leftMargin, sceneLeft, sceneRight, sceneWidth, codeFileCount, codeSize, projectName, sorted) } func max(a, b int) int { diff --git a/render/skyline_test.go b/render/skyline_test.go index 89980a8..94feffa 100644 --- a/render/skyline_test.go +++ b/render/skyline_test.go @@ -22,50 +22,42 @@ func resetSkylineRNG() { rng = rand.New(rand.NewPCG(42, 0)) } -func stripSkylineANSI(s string) string { - return skylineANSIPattern.ReplaceAllString(s, "") +func TestAggregateSkylineFilesFallsBackToAssets(t *testing.T) { + files := []scanner.FileInfo{ + {Path: "images/one.png", Ext: ".png", Size: 10}, + {Path: "images/two.jpg", Ext: ".jpg", Size: 20}, + } + groups, count, size := aggregateSkylineFiles(files) + if count != 2 || size != 30 { + t.Fatalf("fallback totals = (%d, %d), want (2, 30)", count, size) + } + if len(groups) != 2 { + t.Fatalf("fallback groups = %d, want 2", len(groups)) + } } -func TestSkylineFilterCodeFiles(t *testing.T) { - tests := []struct { - name string - files []scanner.FileInfo - expected int - }{ - { - name: "returns only code files when present", - files: []scanner.FileInfo{ - {Path: "main.go", Ext: ".go"}, - {Path: "schema.cue", Ext: ".cue"}, - {Path: "photo.png", Ext: ".png"}, - {Path: "Dockerfile"}, - }, - expected: 3, - }, - { - name: "returns original files when no code files found", - files: []scanner.FileInfo{ - {Path: "image.png", Ext: ".png"}, - {Path: "font.woff", Ext: ".woff"}, - }, - expected: 2, - }, +func TestAggregateSkylineFilesExcludesAssetsFromCodeTotals(t *testing.T) { + files := []scanner.FileInfo{ + {Path: "main.go", Ext: ".go", Size: 10}, + {Path: "images/logo.png", Ext: ".png", Size: 100}, } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := filterCodeFiles(tt.files) - if len(got) != tt.expected { - t.Fatalf("filterCodeFiles() len = %d, want %d", len(got), tt.expected) - } - }) + groups, count, size := aggregateSkylineFiles(files) + if count != 1 || size != 10 { + t.Fatalf("code totals = (%d, %d), want (1, 10)", count, size) + } + if len(groups) != 1 || groups[0].ext != ".go" { + t.Fatalf("code groups = %#v, want only .go", groups) } } +func stripSkylineANSI(s string) string { + return skylineANSIPattern.ReplaceAllString(s, "") +} + func TestSkylineAggregateByExtension(t *testing.T) { files := []scanner.FileInfo{ {Path: "a/main.go", Ext: ".go", Size: 100}, - {Path: "a/util.go", Ext: ".go", Size: 50}, + {Path: "a/util.go", Ext: ".go", Size: 20}, {Path: "b/app.ts", Ext: ".ts", Size: 120}, {Path: "Makefile", Ext: "", Size: 80}, } @@ -75,8 +67,8 @@ func TestSkylineAggregateByExtension(t *testing.T) { t.Fatalf("aggregateByExtension() len = %d, want 3", len(agg)) } - if agg[0].ext != ".go" || agg[0].size != 150 || agg[0].count != 2 { - t.Fatalf("unexpected first aggregate: %+v", agg[0]) + if agg[0].ext != ".go" || agg[0].size != 120 || agg[0].count != 2 || agg[1].ext != ".ts" { + t.Fatalf("first aggregates = %+v, want .go before .ts", agg[:2]) } seenMakefile := false @@ -209,7 +201,7 @@ func TestSkylineRenderStaticIncludesTitleAndStats(t *testing.T) { sorted := []extAgg{{ext: ".go", size: 300, count: 1}} var buf bytes.Buffer - renderStatic(&buf, arranged, 40, 10, 8, 24, 16, codeFiles, "Demo", sorted) + renderStatic(&buf, arranged, 40, 10, 8, 24, 16, len(codeFiles), codeFiles[0].Size, "Demo", sorted) out := buf.String() checks := []string{"─── Demo ───", "1 languages", "1 files", "300.0B"} diff --git a/render/tree.go b/render/tree.go index 58268e8..0d56041 100644 --- a/render/tree.go +++ b/render/tree.go @@ -13,34 +13,43 @@ import ( // treeNode represents a node in the file tree type treeNode struct { - name string - isFile bool - file *scanner.FileInfo - children map[string]*treeNode + name string + isFile bool + file *scanner.FileInfo + children map[string]*treeNode + fileCount int + totalSize int64 + statsReady bool } // getTopLargeFiles returns paths of top 5 largest source code files func getTopLargeFiles(files []scanner.FileInfo) map[string]bool { // Filter out assets and binaries (no extension = likely binary) - var sourceFiles []scanner.FileInfo + var largest []scanner.FileInfo for _, f := range files { ext := strings.ToLower(f.Ext) // Skip if no extension (likely binary) or if it's an asset if ext == "" || IsAssetExtension(ext) { continue } - sourceFiles = append(sourceFiles, f) + position := sort.Search(len(largest), func(i int) bool { + return largest[i].Size < f.Size || largest[i].Size == f.Size && largest[i].Path > f.Path + }) + if position >= 5 { + continue + } + largest = append(largest, scanner.FileInfo{}) + copy(largest[position+1:], largest[position:]) + largest[position] = f + if len(largest) > 5 { + largest = largest[:5] + } } - // Sort by size descending - sort.Slice(sourceFiles, func(i, j int) bool { - return sourceFiles[i].Size > sourceFiles[j].Size - }) - // Return top 5 as set result := make(map[string]bool) - for i := 0; i < len(sourceFiles) && i < 5; i++ { - result[sourceFiles[i].Path] = true + for _, file := range largest { + result[file.Path] = true } return result } @@ -50,44 +59,61 @@ func getDirStats(node *treeNode) (int, int64) { if node.isFile { return 1, node.file.Size } + if node.statsReady { + return node.fileCount, node.totalSize + } count := 0 - var size int64 = 0 + var size int64 for _, child := range node.children { - c, s := getDirStats(child) - count += c - size += s + childCount, childSize := getDirStats(child) + count += childCount + size += childSize } return count, size } +func cacheTreeStats(node *treeNode) (int, int64) { + if node.isFile { + return 1, node.file.Size + } + node.fileCount, node.totalSize = 0, 0 + for _, child := range node.children { + count, size := cacheTreeStats(child) + node.fileCount += count + node.totalSize += size + } + node.statsReady = true + return node.fileCount, node.totalSize +} + // buildTreeStructure builds a nested tree from flat file list func buildTreeStructure(files []scanner.FileInfo) *treeNode { root := &treeNode{children: make(map[string]*treeNode)} - for _, f := range files { - parts := strings.Split(f.Path, string(os.PathSeparator)) + for i := range files { + f := &files[i] current := root - for i, part := range parts { - if i == len(parts)-1 { + remaining := f.Path + for { + separator := strings.IndexByte(remaining, os.PathSeparator) + if separator < 0 { // File - fileCopy := f - current.children[part] = &treeNode{ - name: part, + current.children[remaining] = &treeNode{ + name: remaining, isFile: true, - file: &fileCopy, + file: f, } - } else { - // Directory - if current.children[part] == nil { - current.children[part] = &treeNode{ - name: part, - children: make(map[string]*treeNode), - } - } - current = current.children[part] + break + } + part := remaining[:separator] + if current.children[part] == nil { + current.children[part] = &treeNode{name: part, children: make(map[string]*treeNode)} } + current = current.children[part] + remaining = remaining[separator+1:] } } + cacheTreeStats(root) return root } diff --git a/render/tree_test.go b/render/tree_test.go index 3271c90..991004d 100644 --- a/render/tree_test.go +++ b/render/tree_test.go @@ -4,7 +4,7 @@ import ( "bytes" "context" "math/rand/v2" - "reflect" + "path/filepath" "strings" "testing" @@ -235,6 +235,27 @@ func TestGetTopLargeFilesFewerThan5(t *testing.T) { } } +func TestGetTopLargeFilesBreaksSizeTiesByPath(t *testing.T) { + files := []scanner.FileInfo{ + {Path: "z.go", Size: 100, Ext: ".go"}, + {Path: "f.go", Size: 100, Ext: ".go"}, + {Path: "a.go", Size: 100, Ext: ".go"}, + {Path: "b.go", Size: 100, Ext: ".go"}, + {Path: "c.go", Size: 100, Ext: ".go"}, + {Path: "d.go", Size: 100, Ext: ".go"}, + {Path: "e.go", Size: 100, Ext: ".go"}, + } + top := getTopLargeFiles(files) + for _, path := range []string{"a.go", "b.go", "c.go", "d.go", "e.go"} { + if !top[path] { + t.Fatalf("top files = %v, want %s", top, path) + } + } + if top["f.go"] || top["z.go"] { + t.Fatalf("top files include a larger tie-break path: %v", top) + } +} + func TestTreeNodeStructure(t *testing.T) { // Test treeNode creation node := &treeNode{ @@ -260,6 +281,17 @@ func TestTreeNodeStructure(t *testing.T) { } } +func TestBuildTreeStructureCountsDuplicatePathOnce(t *testing.T) { + root := buildTreeStructure([]scanner.FileInfo{ + {Path: filepath.Join("src", "main.go"), Size: 10}, + {Path: filepath.Join("src", "main.go"), Size: 20}, + }) + count, size := getDirStats(root.children["src"]) + if count != 1 || size != 20 { + t.Fatalf("duplicate path stats = (%d, %d), want (1, 20)", count, size) + } +} + func TestTitleCase(t *testing.T) { tests := []struct { name string @@ -303,48 +335,6 @@ func TestGetSystemName(t *testing.T) { } } -func TestFilterCodeFiles(t *testing.T) { - tests := []struct { - name string - files []scanner.FileInfo - want []scanner.FileInfo - }{ - { - name: "filters to code extensions and known code filenames", - files: []scanner.FileInfo{ - {Path: "main.go", Ext: ".go"}, - {Path: "README.md", Ext: ".md"}, - {Path: "Dockerfile", Ext: ""}, - {Path: "assets/logo.png", Ext: ".png"}, - }, - want: []scanner.FileInfo{ - {Path: "main.go", Ext: ".go"}, - {Path: "Dockerfile", Ext: ""}, - }, - }, - { - name: "returns original slice when no code files match", - files: []scanner.FileInfo{ - {Path: "README.md", Ext: ".md"}, - {Path: "assets/logo.png", Ext: ".png"}, - }, - want: []scanner.FileInfo{ - {Path: "README.md", Ext: ".md"}, - {Path: "assets/logo.png", Ext: ".png"}, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := filterCodeFiles(tt.files) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("filterCodeFiles() = %#v, want %#v", got, tt.want) - } - }) - } -} - func TestAggregateByExtension(t *testing.T) { files := []scanner.FileInfo{ {Path: "main.go", Ext: ".go", Size: 100}, From b88c5eaa1138025ad7e50a545d3f81b5db592457 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:21:46 +0200 Subject: [PATCH 7/7] perf(watch): Bound daemon hot paths Reuse resolved policy paths and startup inventories to avoid repeated worktree discovery and directory walks. Stream state directly into atomic replacements while preserving the previous state on encoding failure. --- config/config.go | 7 ++- config/config_test.go | 35 +++++++++++++- internal/runtimefile/runtimefile.go | 11 ++++- internal/runtimefile/runtimefile_test.go | 59 ++++++++++++++++++++++++ scanner/walker.go | 9 +++- scanner/walker_test.go | 8 ++++ watch/daemon.go | 57 +++++++++++++++-------- watch/events.go | 13 +++--- watch/graph_state_test.go | 2 +- watch/publication.go | 12 ++--- watch/state_test.go | 16 +++++++ 11 files changed, 193 insertions(+), 36 deletions(-) create mode 100644 internal/runtimefile/runtimefile_test.go diff --git a/config/config.go b/config/config.go index d3e6662..656c16b 100644 --- a/config/config.go +++ b/config/config.go @@ -234,7 +234,12 @@ func ConfigPath(root string) string { // Returns zero-value ProjectConfig if the file is missing. // Logs a warning to stderr and returns zero-value if JSON is malformed. func Load(root string) ProjectConfig { - data, err := os.ReadFile(ConfigPath(root)) + return LoadFile(ConfigPath(root)) +} + +// LoadFile reads a previously resolved config path. +func LoadFile(path string) ProjectConfig { + data, err := os.ReadFile(path) if err != nil { return ProjectConfig{} } diff --git a/config/config_test.go b/config/config_test.go index b3a9bd7..7721c17 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -97,6 +97,18 @@ func TestLoad_ValidConfig(t *testing.T) { } } +func TestLoadFileReadsResolvedConfigPath(t *testing.T) { + path := filepath.Join(t.TempDir(), "policy.json") + if err := os.WriteFile(path, []byte(`{"only":["go"],"depth":2}`), 0o644); err != nil { + t.Fatal(err) + } + + cfg := LoadFile(path) + if len(cfg.Only) != 1 || cfg.Only[0] != "go" || cfg.Depth != 2 { + t.Fatalf("LoadFile() = %+v", cfg) + } +} + func TestLoad_PartialConfig(t *testing.T) { dir := t.TempDir() codemapDir := filepath.Join(dir, ".codemap") @@ -198,7 +210,7 @@ func TestConfigPathAndLoadUseSelectedSetupRoot(t *testing.T) { } } -func makeLinkedConfigWorktreeFixture(t *testing.T) (primary, linked string) { +func makeLinkedConfigWorktreeFixture(t testing.TB) (primary, linked string) { t.Helper() primary = t.TempDir() gitDir := filepath.Join(primary, ".git", "worktrees", "agent") @@ -226,6 +238,27 @@ func makeLinkedConfigWorktreeFixture(t *testing.T) (primary, linked string) { return primary, linked } +func BenchmarkLoadLinkedWorktreeConfig(b *testing.B) { + projectpath.ResetSetupRoot() + b.Cleanup(projectpath.ResetSetupRoot) + primary, linked := makeLinkedConfigWorktreeFixture(b) + path := filepath.Join(primary, ".codemap", "config.json") + if err := os.WriteFile(path, []byte(`{"only":["go"],"exclude":["vendor"]}`), 0o644); err != nil { + b.Fatal(err) + } + + b.Run("resolve-root", func(b *testing.B) { + for b.Loop() { + _ = Load(linked) + } + }) + b.Run("resolved-path", func(b *testing.B) { + for b.Loop() { + _ = LoadFile(path) + } + }) +} + func TestPolicyDefaultsAndClamps(t *testing.T) { t.Run("defaults for empty config", func(t *testing.T) { var cfg ProjectConfig diff --git a/internal/runtimefile/runtimefile.go b/internal/runtimefile/runtimefile.go index af5f58f..1fe63bb 100644 --- a/internal/runtimefile/runtimefile.go +++ b/internal/runtimefile/runtimefile.go @@ -2,12 +2,21 @@ package runtimefile import ( "fmt" + "io" "os" "path/filepath" ) // WriteAtomic replaces a regular runtime file without following its endpoint. func WriteAtomic(path string, data []byte, mode os.FileMode) error { + return WriteAtomicWith(path, mode, func(w io.Writer) error { + _, err := w.Write(data) + return err + }) +} + +// WriteAtomicWith replaces a regular runtime file with streamed content. +func WriteAtomicWith(path string, mode os.FileMode, write func(io.Writer) error) error { if info, err := os.Lstat(path); err == nil && !info.Mode().IsRegular() { return fmt.Errorf("unsafe runtime file %q", path) } else if err != nil && !os.IsNotExist(err) { @@ -23,7 +32,7 @@ func WriteAtomic(path string, data []byte, mode os.FileMode) error { _ = tmp.Close() return err } - if _, err := tmp.Write(data); err != nil { + if err := write(tmp); err != nil { _ = tmp.Close() return err } diff --git a/internal/runtimefile/runtimefile_test.go b/internal/runtimefile/runtimefile_test.go new file mode 100644 index 0000000..f566d25 --- /dev/null +++ b/internal/runtimefile/runtimefile_test.go @@ -0,0 +1,59 @@ +package runtimefile + +import ( + "errors" + "io" + "os" + "path/filepath" + "testing" +) + +func TestWriteAtomicWithPreservesDestinationAfterWriteFailure(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "state.json") + if err := os.WriteFile(path, []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + wantErr := errors.New("encode failed") + err := WriteAtomicWith(path, 0o644, func(w io.Writer) error { + _, _ = w.Write([]byte("partial")) + return wantErr + }) + if !errors.Is(err, wantErr) { + t.Fatalf("error = %v, want %v", err, wantErr) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(data) != "old" { + t.Fatalf("destination = %q, want old", data) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Fatalf("directory entries = %d, want 1", len(entries)) + } +} + +func TestWriteAtomicWithReplacesDestination(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + if err := os.WriteFile(path, []byte("old"), 0o600); err != nil { + t.Fatal(err) + } + if err := WriteAtomicWith(path, 0o644, func(w io.Writer) error { + _, err := io.WriteString(w, "new") + return err + }); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(data) != "new" { + t.Fatalf("content = %q, want new", data) + } +} diff --git a/scanner/walker.go b/scanner/walker.go index 40042a3..b10c7c0 100644 --- a/scanner/walker.go +++ b/scanner/walker.go @@ -326,11 +326,16 @@ func ConfiguredFilters(root string) Filters { // ScanConfiguredFiles scans using the active setup root's project filters // while honoring caller cancellation. func ScanConfiguredFiles(ctx context.Context, root string, cache *GitIgnoreCache) ([]FileInfo, error) { + cfg := config.Load(root) + return ScanConfiguredFilesWithFilters(ctx, root, cache, Filters{Only: cfg.Only, Exclude: cfg.Exclude}) +} + +// ScanConfiguredFilesWithFilters scans files using already resolved filters. +func ScanConfiguredFilesWithFilters(ctx context.Context, root string, cache *GitIgnoreCache, filters Filters) ([]FileInfo, error) { if err := ctx.Err(); err != nil { return nil, err } - cfg := config.Load(root) - files, err := ScanFiles(ctx, root, cache, cfg.Only, cfg.Exclude) + files, err := ScanFiles(ctx, root, cache, filters.Only, filters.Exclude) if err != nil { return nil, err } diff --git a/scanner/walker_test.go b/scanner/walker_test.go index e58f319..b45f025 100644 --- a/scanner/walker_test.go +++ b/scanner/walker_test.go @@ -825,6 +825,14 @@ func TestScanConfiguredFilesExcludesCodemapState(t *testing.T) { if !reflect.DeepEqual(paths, want) { t.Fatalf("configured files = %v, want %v (no .codemap entries)", paths, want) } + + files, err = ScanConfiguredFilesWithFilters(context.Background(), root, NewGitIgnoreCache(root), Filters{Only: []string{"go"}, Exclude: []string{"pkg"}}) + if err != nil { + t.Fatal(err) + } + if len(files) != 1 || filepath.ToSlash(files[0].Path) != "main.go" { + t.Fatalf("explicitly filtered files = %v, want main.go", files) + } } func TestFilterAnalysesContextBranches(t *testing.T) { diff --git a/watch/daemon.go b/watch/daemon.go index 12f83bd..18fcd98 100644 --- a/watch/daemon.go +++ b/watch/daemon.go @@ -30,6 +30,7 @@ var ( type Daemon struct { root string configDir string + configPath string runtimeDir string graph *Graph watcher *fsnotify.Watcher @@ -73,6 +74,16 @@ func (d *Daemon) runtimeStateDir() (string, error) { return projectpath.CheckedRuntimeCodemapDir(d.root) } +func (d *Daemon) loadConfig() config.ProjectConfig { + if d.configPath != "" { + return config.LoadFile(d.configPath) + } + if d.configDir != "" { + return config.LoadFile(filepath.Join(d.configDir, "config.json")) + } + return config.Load(d.root) +} + func (d *Daemon) ensurePublisher() error { if d.publisher != nil { return nil @@ -113,6 +124,7 @@ func NewDaemon(root string, verbose bool) (*Daemon, error) { d := &Daemon{ root: absRoot, configDir: selection.PolicyDir, + configPath: filepath.Join(selection.PolicyDir, "config.json"), runtimeDir: runtimeDir, watcher: watcher, gitCache: gitCache, @@ -312,19 +324,24 @@ func (d *Daemon) WriteInitialState() { // fullScan does a complete scan of the project func (d *Daemon) fullScan() error { start := time.Now() + cfg := d.loadConfig() files, err := scanner.ScanFiles(context.Background(), d.root, d.gitCache, nil, nil) if err != nil { return err } - configuredFiles, err := scanner.ScanConfiguredFiles(context.Background(), d.root, d.gitCache) - if err != nil { - return err + configuredPaths := make([]string, 0) + for i := range files { + file := &files[i] + path := filepath.ToSlash(file.Path) + if path != ".codemap" && !strings.HasPrefix(path, ".codemap/") && scanner.MatchesFilters(file.Path, file.Ext, cfg.Only, cfg.Exclude) { + configuredPaths = append(configuredPaths, file.Path) + } } d.graph.mu.Lock() d.graph.Files = make(map[string]*scanner.FileInfo) - d.graph.ConfiguredFiles = make(map[string]struct{}, len(configuredFiles)) + d.graph.ConfiguredFiles = make(map[string]struct{}, len(configuredPaths)) d.graph.State = make(map[string]*FileState) for i := range files { f := &files[i] @@ -334,8 +351,8 @@ func (d *Daemon) fullScan() error { d.graph.State[f.Path] = &FileState{Lines: lines, Size: f.Size} } } - for _, file := range configuredFiles { - d.graph.ConfiguredFiles[file.Path] = struct{}{} + for _, path := range configuredPaths { + d.graph.ConfiguredFiles[path] = struct{}{} } d.graph.LastScan = time.Now() d.graph.mu.Unlock() @@ -347,8 +364,7 @@ func (d *Daemon) fullScan() error { return nil } -func (d *Daemon) isConfiguredFile(path string) bool { - cfg := config.Load(d.root) +func matchesConfiguredFile(path string, cfg config.ProjectConfig) bool { return scanner.MatchesFilters(path, filepath.Ext(path), cfg.Only, cfg.Exclude) } @@ -360,7 +376,8 @@ func (d *Daemon) refreshConfiguredFiles(resetIgnoreCache bool) error { gitCache = scanner.NewGitIgnoreCache(d.root) d.gitCache = gitCache } - files, err := scanner.ScanConfiguredFiles(context.Background(), d.root, gitCache) + cfg := d.loadConfig() + files, err := scanner.ScanConfiguredFilesWithFilters(context.Background(), d.root, gitCache, scanner.Filters{Only: cfg.Only, Exclude: cfg.Exclude}) if err != nil { return err } @@ -372,7 +389,7 @@ func (d *Daemon) refreshConfiguredFiles(resetIgnoreCache bool) error { d.graph.ConfiguredFiles = configured // Filters define dependency membership too, so the previous graph must not // be published under a new configured-file count. - d.markGraphLifecycleLocked(newGraphState(d.root, config.Load(d.root), graphLifecycleStale, time.Time{}, nil)) + d.markGraphLifecycleLocked(newGraphState(d.root, cfg, graphLifecycleStale, time.Time{}, nil)) d.graph.mu.Unlock() // Invalidation alone would leave the daemon serving no hub or importer @@ -390,6 +407,7 @@ var daemonRefreshDependencies = (*Daemon).refreshDependencies // refreshDependencies is called by eventLoop and owns the worker state flags. func (d *Daemon) refreshDependencies() { + cfg := d.loadConfig() d.graph.mu.RLock() stale := d.graph.GraphState.Status == graphLifecycleStale configuredCount := len(d.graph.ConfiguredFiles) @@ -399,7 +417,7 @@ func (d *Daemon) refreshDependencies() { } snapshot := dependencyGraphSnapshot{ configured: configured, - config: config.Load(d.root), + config: cfg, generation: d.graph.graphGeneration, } d.graph.mu.RUnlock() @@ -421,13 +439,13 @@ func (d *Daemon) refreshDependencies() { // buildDependencyGraph converts a worker panic into the existing failed-build // path so a background scan cannot terminate the daemon process. -func buildDependencyGraph(ctx context.Context, root string, build func(context.Context, string, scanner.Filters) (*scanner.FileGraph, error)) (graph *scanner.FileGraph, err error) { +func buildDependencyGraph(ctx context.Context, root string, filters scanner.Filters, build func(context.Context, string, scanner.Filters) (*scanner.FileGraph, error)) (graph *scanner.FileGraph, err error) { defer func() { if recovered := recover(); recovered != nil { err = fmt.Errorf("dependency graph build panicked: %v", recovered) } }() - return build(ctx, root, scanner.ConfiguredFilters(root)) + return build(ctx, root, filters) } func (d *Daemon) startDependencyWorker() { @@ -449,7 +467,8 @@ func (d *Daemon) startDependencyWorker() { return case snapshot := <-d.dependencyRequests: started := time.Now() - graph, err := buildDependencyGraph(ctx, d.root, buildFileGraph) + filters := scanner.Filters{Only: snapshot.config.Only, Exclude: snapshot.config.Exclude} + graph, err := buildDependencyGraph(ctx, d.root, filters, buildFileGraph) result := dependencyGraphResult{snapshot: snapshot, graph: graph, err: err, started: started} select { case d.dependencyResults <- result: @@ -469,9 +488,10 @@ func (d *Daemon) handleDependencyGraphResult(result dependencyGraphResult) { retry := d.dependencyPending d.dependencyPending = false if retry { + cfg := d.loadConfig() d.graph.mu.Lock() if d.graph.GraphState.Status != graphLifecycleStale { - d.markGraphLifecycleLocked(newGraphState(d.root, config.Load(d.root), graphLifecycleStale, time.Time{}, nil)) + d.markGraphLifecycleLocked(newGraphState(d.root, cfg, graphLifecycleStale, time.Time{}, nil)) } d.graph.mu.Unlock() } @@ -508,6 +528,7 @@ func (d *Daemon) computeDepsWithBeforePublish(build func(context.Context, string } func (d *Daemon) dependencyGraphSnapshot() dependencyGraphSnapshot { + cfg := d.loadConfig() d.graph.mu.RLock() defer d.graph.mu.RUnlock() configured := make([]string, 0, len(d.graph.ConfiguredFiles)) @@ -516,7 +537,7 @@ func (d *Daemon) dependencyGraphSnapshot() dependencyGraphSnapshot { } return dependencyGraphSnapshot{ configured: configured, - config: config.Load(d.root), + config: cfg, generation: d.graph.graphGeneration, } } @@ -530,13 +551,13 @@ func (d *Daemon) applyDependencyGraph(snapshot dependencyGraphSnapshot, fg *scan return } + currentConfig := d.loadConfig() d.graph.mu.Lock() defer d.graph.mu.Unlock() configuredAfter := make([]string, 0, len(d.graph.ConfiguredFiles)) for file := range d.graph.ConfiguredFiles { configuredAfter = append(configuredAfter, file) } - currentConfig := config.Load(d.root) if d.graph.graphGeneration != snapshot.generation || ConfiguredInventoryFingerprint(snapshot.configured) != ConfiguredInventoryFingerprint(configuredAfter) || graphFilterFingerprint(snapshot.config) != graphFilterFingerprint(currentConfig) { @@ -567,7 +588,7 @@ func (d *Daemon) applyDependencyGraph(snapshot dependencyGraphSnapshot, fg *scan } func (d *Daemon) markGraphLifecycle(status GraphLifecycle) { - state := newGraphState(d.root, config.Load(d.root), status, time.Time{}, nil) + state := newGraphState(d.root, d.loadConfig(), status, time.Time{}, nil) d.graph.mu.Lock() defer d.graph.mu.Unlock() d.markGraphLifecycleLocked(state) diff --git a/watch/events.go b/watch/events.go index d482bad..21c0c4f 100644 --- a/watch/events.go +++ b/watch/events.go @@ -11,7 +11,6 @@ import ( "strings" "time" - "codemap/config" "codemap/internal/projectpath" "codemap/internal/runtimefile" "codemap/limits" @@ -333,6 +332,7 @@ func (d *Daemon) filterControlEvent(path string) (resetIgnoreCache, control bool func (d *Daemon) handleConfiguredMembershipEvent(event fsnotify.Event) bool { event.Name = projectpath.CanonicalPath(event.Name) + cfg := d.loadConfig() relPath, err := filepath.Rel(projectpath.CanonicalPath(d.root), event.Name) if err != nil { return false @@ -343,7 +343,7 @@ func (d *Daemon) handleConfiguredMembershipEvent(event fsnotify.Event) bool { present := event.Op&(fsnotify.Remove|fsnotify.Rename) == 0 if present { info, err := os.Stat(event.Name) - if err != nil || info.IsDir() || (d.gitCache != nil && d.gitCache.ShouldIgnore(event.Name)) || !d.isConfiguredFile(relPath) { + if err != nil || info.IsDir() || (d.gitCache != nil && d.gitCache.ShouldIgnore(event.Name)) || !matchesConfiguredFile(relPath, cfg) { present = false } } @@ -356,7 +356,7 @@ func (d *Daemon) handleConfiguredMembershipEvent(event fsnotify.Event) bool { } changed := present != existed if changed { - d.markGraphLifecycleLocked(newGraphState(d.root, config.Load(d.root), graphLifecycleStale, time.Time{}, nil)) + d.markGraphLifecycleLocked(newGraphState(d.root, cfg, graphLifecycleStale, time.Time{}, nil)) } d.graph.mu.Unlock() if changed { @@ -537,6 +537,7 @@ func (d *Daemon) handleEvent(fsEvent fsnotify.Event) bool { default: return false } + cfg := d.loadConfig() event := Event{ Time: time.Now(), @@ -562,7 +563,7 @@ func (d *Daemon) handleEvent(fsEvent fsnotify.Event) bool { delete(d.graph.ConfiguredFiles, relPath) delete(d.graph.State, relPath) if wasConfigured { - state := newGraphState(d.root, config.Load(d.root), graphLifecycleStale, time.Time{}, nil) + state := newGraphState(d.root, cfg, graphLifecycleStale, time.Time{}, nil) d.markGraphLifecycleLocked(state) } } @@ -621,7 +622,7 @@ func (d *Daemon) handleEvent(fsEvent fsnotify.Event) bool { if d.graph.ConfiguredFiles == nil { d.graph.ConfiguredFiles = make(map[string]struct{}) } - if d.isConfiguredFile(relPath) { + if matchesConfiguredFile(relPath, cfg) { d.graph.ConfiguredFiles[relPath] = struct{}{} isConfigured = true } else { @@ -641,7 +642,7 @@ func (d *Daemon) handleEvent(fsEvent fsnotify.Event) bool { } graphInvalidated := wasConfigured || isConfigured if graphInvalidated { - state := newGraphState(d.root, config.Load(d.root), graphLifecycleStale, time.Time{}, nil) + state := newGraphState(d.root, cfg, graphLifecycleStale, time.Time{}, nil) d.markGraphLifecycleLocked(state) } diff --git a/watch/graph_state_test.go b/watch/graph_state_test.go index b0200b4..235b421 100644 --- a/watch/graph_state_test.go +++ b/watch/graph_state_test.go @@ -220,7 +220,7 @@ func TestDependencyRefreshBuildsOffEventLoop(t *testing.T) { } func TestDependencyGraphBuildRecoversPanics(t *testing.T) { - graph, err := buildDependencyGraph(context.Background(), t.TempDir(), func(context.Context, string, scanner.Filters) (*scanner.FileGraph, error) { + graph, err := buildDependencyGraph(context.Background(), t.TempDir(), scanner.Filters{}, func(context.Context, string, scanner.Filters) (*scanner.FileGraph, error) { panic("test panic") }) if graph != nil || err == nil || err.Error() != "dependency graph build panicked: test panic" { diff --git a/watch/publication.go b/watch/publication.go index 6ca8c79..04793d4 100644 --- a/watch/publication.go +++ b/watch/publication.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "io" "os" "path/filepath" "slices" @@ -90,13 +91,12 @@ func (p *statePublisher) snapshot(generation uint64) State { func (p *statePublisher) publish() error { next := p.generation + 1 - data, err := json.MarshalIndent(p.snapshot(next), "", " ") + err := runtimefile.WriteAtomicWith(p.path, 0o644, func(w io.Writer) error { + encoder := json.NewEncoder(w) + encoder.SetIndent("", " ") + return encoder.Encode(p.snapshot(next)) + }) if err != nil { - p.dirty = true - p.deadline = time.Now().Add(publicationRetryDelay) - return err - } - if err = runtimefile.WriteAtomic(p.path, data, 0o644); err != nil { p.dirty = true ackErr := p.failPending("publication_failed") p.deadline = time.Now().Add(publicationRetryDelay) diff --git a/watch/state_test.go b/watch/state_test.go index 992b5d6..14d82f9 100644 --- a/watch/state_test.go +++ b/watch/state_test.go @@ -13,6 +13,8 @@ import ( "codemap/internal/projectpath" "codemap/scanner" + + "github.com/fsnotify/fsnotify" ) func TestHelperWatchDaemonProcess(t *testing.T) { @@ -434,6 +436,9 @@ func TestAutomaticLinkedWorktreeUsesLocalWatchStorage(t *testing.T) { if err := os.MkdirAll(filepath.Join(primary, ".codemap"), 0o755); err != nil { t.Fatal(err) } + if err := os.WriteFile(filepath.Join(primary, ".codemap", "config.json"), []byte(`{"only":["go"]}`), 0o644); err != nil { + t.Fatal(err) + } if err := os.WriteFile(filepath.Join(gitDir, "commondir"), []byte("../..\n"), 0o644); err != nil { t.Fatal(err) } @@ -471,6 +476,17 @@ func TestAutomaticLinkedWorktreeUsesLocalWatchStorage(t *testing.T) { if _, err := os.Stat(filepath.Join(projectpath.ProjectRuntimeDir(primary), "state.json")); !os.IsNotExist(err) { t.Fatalf("primary state unexpectedly created: %v", err) } + + if err := os.WriteFile(filepath.Join(linked, ".git"), []byte("invalid\n"), 0o644); err != nil { + t.Fatal(err) + } + textFile := filepath.Join(linked, "notes.txt") + if err := os.WriteFile(textFile, []byte("not configured\n"), 0o644); err != nil { + t.Fatal(err) + } + if changed := d.handleConfiguredMembershipEvent(fsnotify.Event{Name: textFile, Op: fsnotify.Create}); changed { + t.Fatal("linked worktree stopped using its resolved primary policy") + } } func TestProcessAliveDetectsLiveAndDeadPIDs(t *testing.T) {