diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 4f1832ff3..0de9f089c 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -1,5 +1,37 @@ # Testing Guide +`Extract_CSharpStaticLambdaGate_BoundsRepeatedSameLineDeclarations` checks 64 +same-line static methods with a warmed 2 MiB allocation ceiling on both runtimes, +including C#, Razor, Blazor and CSHTML. Preserve all identities, raw start columns, +return types and expression-body ranges. Static constructors and generic methods +must survive; typed, tuple-return, function-pointer-return and inferred static +lambdas must not create phantom declarations. Run the existing #4830 query corpus +and the full symbol/reference extractor regressions alongside this fixture. +`Extract_CSharpMethodConfirmation_RequiresOpeningParenthesis` compares every +symbol property with optimizations disabled and requires fewer method-confirmation +regex attempts for parameter fragments without `(`, while real multiline headers +still reach the regex. Keep the attempt/skip counters internal to extraction tests. + +Confirmed C# method lookahead coverage in `SymbolExtractorCSharpRegexProbeTests` +shares multiline methods, empty and nonempty constructor bodies, delayed body +tokens, trailing fields and attributed accessors across C#, Razor, Blazor and +CSHTML. Keep the warmed 512 KiB allocation ceiling, all declaration identities, +signatures and body ranges on net8/net9. `DatabaseTests` verifies that both +timestamp and stat reuse reject old C# contract 18 / alias contract 1 and resume +after stamping contract 19. The full-scan upgrade regressions remain required. + +Initial fresh-source lookup coverage in `AuthoritativeFreshRawBulkInsertTests` +shares canonical/display/legacy names, nested and tied ranges, out-of-range +candidates and multi-file controls across C#, Python, JavaScript, TypeScript, +Java, Go, Rust, C++, Kotlin and VB. Preserve the single final three-candidate +sort and the 32-callback budget (1,000 SQLite VM instructions each) for 64 +lookups over 128 overlapping declarations, alongside savepoint and cancellation +regressions. Run the suite on net8/net9; timing thresholds are not used here. +Include `FreshReferenceResolutionTests` in that run: it covers direct and +materialized source lookup, the shared 14-parameter row shape and separate SQL +caches. The materialized outer comparison orders by the projected `start_line`; +each probe applies the nullable-line fallback before selecting its candidate. + #5339 extends the #5332 fixture to successful `files --format count --json` batches with the same three-snapshot/copy-byte budget for 3/12 items, checking counts, freshness and authority fields. Keep metadata-present/absent controls diff --git a/changelog.d/unreleased/+initial-index-declaration-probes.changed.md b/changelog.d/unreleased/+initial-index-declaration-probes.changed.md new file mode 100644 index 000000000..4b0bfec05 --- /dev/null +++ b/changelog.d/unreleased/+initial-index-declaration-probes.changed.md @@ -0,0 +1,15 @@ +--- +category: changed +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs + - tests/CodeIndex.Tests/SymbolExtractorCSharpRegexProbeTests.cs +--- + +## English + +- **Bound declaration probes during full indexing** — C#, Razor, Blazor and CSHTML avoid rechecking static-lambda arrows and prefixes across unrelated declarations, and reject impossible method prefixes before regex matching. Static constructors, generic methods and lambda filtering retain their results, with fewer regex attempts and substantially fewer temporary allocations. + +## 日本語 + +- **フルインデックス時の宣言候補の検査範囲を制限** — C#・Razor・Blazor・CSHTML で別の宣言に跨る静的 lambda の arrow・prefix の再検査を避け、成立しないメソッド prefix も regex 照合の前に除外します。static constructor・generic method・lambda 除外の結果を保ちながら、regex 試行回数と一時割り当てを減らします。 diff --git a/changelog.d/unreleased/+initial-index-method-lookahead.changed.md b/changelog.d/unreleased/+initial-index-method-lookahead.changed.md new file mode 100644 index 000000000..5bc38b8f4 --- /dev/null +++ b/changelog.d/unreleased/+initial-index-method-lookahead.changed.md @@ -0,0 +1,14 @@ +--- +category: changed +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Contracts.cs +--- + +## English + +- **Bound confirmed method lookahead during indexing** — C#, Razor, Blazor and CSHTML stop merging method bodies once an accessor is ruled out, reducing temporary allocations in large files while preserving ranges and recovering affected constructors. Extractor contract 19 refreshes existing rows on the next ordinary full scan. + +## 日本語 + +- **インデックス作成時の確認済みメソッドの先読みを制限** — C#・Razor・Blazor・CSHTML で accessor でないと判明した時点で body の連結を止め、大きいファイルの一時割り当てを減らします。範囲を維持し、影響を受けていた constructor も回復します。次の通常フルスキャンでは抽出契約19により既存行も更新します。 diff --git a/changelog.d/unreleased/+initial-index-ranked-source-probes.changed.md b/changelog.d/unreleased/+initial-index-ranked-source-probes.changed.md new file mode 100644 index 000000000..b99b3914e --- /dev/null +++ b/changelog.d/unreleased/+initial-index-ranked-source-probes.changed.md @@ -0,0 +1,14 @@ +--- +category: changed +affected: + - src/CodeIndex/Database/DbWriter.AuthoritativeFreshReferenceSourceLookup.cs + - tests/CodeIndex.Tests/AuthoritativeFreshRawBulkInsertTests.cs +--- + +## English + +- **Reduce repeated source-reference sorting during initial full indexing** — temporary name indexes now select the best containing declaration before comparing at most three candidates. All languages retain the same source identities, alias and legacy-name behavior, and transactional recovery. + +## 日本語 + +- **初回フルインデックスでの参照元候補の反復ソートを削減** — 一時的な名前 index で最適な包含宣言を先に選び、最後の比較を最大3候補に制限します。全言語で参照元の同一性、別名・旧形式の名前照合、トランザクションの復旧動作を維持します。 diff --git a/docs/initial-index-performance.md b/docs/initial-index-performance.md index f9f5b2705..18976e7d8 100644 --- a/docs/initial-index-performance.md +++ b/docs/initial-index-performance.md @@ -20,6 +20,15 @@ ranked ID is consumed. Avoiding duplicate elimination removes a temporary B-tree per reference while preserving same-file scoping, nested range selection, tie-breaking, and legacy fallback. Ordinary source repair remains unchanged. +Those temporary indexes also carry containment rank. Each probe stops at its +first containing declaration, and the final comparison sorts at most three +candidates. A non-containing early candidate is skipped before that limit. +This applies to every language using the fresh writer and retains file-savepoint +rollback. A regression with 128 overlapping same-name declarations and 64 lookups +used 520 callbacks at 1,000 SQLite VM instructions per callback before this +change; the new budget is at most 32 callbacks on both test runtimes. This measures +lookup work, not end-to-end speed or worst-case misses across disjoint ranges. + All language-independent folded symbol/reference keys use runtime-vectorized ASCII validation and casing. Unicode names still normalize with NFKC and apply the same vendored casefold deltas, but append lowered scalar values directly @@ -27,6 +36,41 @@ instead of allocating intermediate strings per character. Already folded ASCII names retain their original string. Persisted keys, their contract version and runtime fingerprint are unchanged; readers and every writer use the same fold. +C# declaration lookahead retains a confirmed method header while checking for +property accessors. When the first body token rules out an accessor, it stops +there instead of joining subsequent method bodies up to a later field. This +avoids repeated large strings in files with many multiline methods, preserves +method ranges and signatures, and recovers affected constructor declarations. +The same extractor handles Razor, Blazor and CSHTML. All four language keys now +use C# extractor contract 19, so an ordinary full scan refreshes old rows even +when source files are unchanged. Comments and accessor attributes still receive +the existing lookahead. + +C# static-lambda rejection examines only arrows that can still enclose the +candidate name and explicit-return prefixes within the current declaration +segment. Parenthesized tuple groups remain intact. This avoids repeated scans +and prefix allocations across earlier members on a dense line, while preserving +static constructors, generic methods, and typed/tuple/function-pointer lambdas. +Razor, Blazor and CSHTML share this behavior. A warmed extraction of 64 same-line +static methods allocated 406.5 MB before these bounds and 0.93 MB after them; +the regression fixture includes positive controls and allows 2 MiB on net8/net9. +This allocation budget does not assert a wall-clock speed for other inputs. + +Confirmed method-prefix matching also requires an opening parenthesis before +running the regex. Parameter continuation fragments ending in `) {` cannot match +without it; the fast rejection preserves full symbol records and is covered by +regex-attempt counts rather than a timing threshold. + +On a fixed 1,554-file snapshot of this repository at `0d39d0658`, a fresh Release +.NET 8 index on macOS ARM64 with `--parallelism 2 --memory-trace` took 76.0 seconds +before these three changes and 68.9 seconds after them. Total managed allocations +fell from 18.46 GB to 7.83 GB. Each run used a new database and the same source +snapshot. These are single-run observations, not a general speed guarantee. +All pre-existing symbol identities and extraction metadata remained identical; +seven missed constructors were recovered, removing seven declaration-site +reference rows and updating the corresponding overload-resolution candidates. +Both runs completed all 1,554 files without warnings or extraction errors. + ## 日本語 空のデータベースに対する通常の CLI フルスキャンは、初回専用の一括 writer を @@ -46,8 +90,46 @@ DB レイアウト、抽出範囲、トランザクション境界、取消・ を省きつつ、同一ファイルへの限定、入れ子の選択、同順位の決定、旧形式への fallback を維持します。通常の参照元修復処理は変更しません。 +一時 index には包含範囲の順位も持たせ、各照会では参照行を含む最上位の宣言だけを +取得します。最後に並べ替える候補は最大3件です。範囲外の候補は件数制限の前に除外 +します。初回 writer を使う全言語に適用し、ファイル単位の savepoint rollback も +維持します。同名の重複範囲128件に対する64回の照会では、変更前は SQLite VM の +1,000命令ごとの callback が520回、変更後の回帰テスト上限は両 runtime で32回です。 +これは照会処理量の測定であり、全体の速度や互いに離れた範囲での miss の最悪値を +保証するものではありません。 + 全言語共通のシンボル・参照の folded key は、ランタイムのベクトル化された ASCII 検査と小文字化を使います。Unicode 名は既存の NFKC 正規化と casefold 差分表を 維持しつつ、文字ごとの一時文字列を作らずに小文字化した scalar を直接追加します。 変換済みの ASCII 名は元の文字列を再利用します。保存 key、契約バージョン、実行環境 fingerprint は変わらず、reader と各 writer は同じ変換を使います。 + +C# 宣言の先読みでは、プロパティの accessor を調べる間、確認済みのメソッド +header を保持します。body の先頭 token で accessor でないと分かれば停止し、 +後続メソッドの body を末尾の field まで連結し続けません。複数行メソッドが多い +ファイルで巨大な一時文字列の反復生成を避け、メソッドの範囲・signature を保ち、 +影響を受けていた constructor 宣言も回復します。同じ抽出器を使う Razor・Blazor・ +CSHTML を含む4つの言語 key は C# 抽出契約19を使い、通常のフルスキャンで未変更の +既存ファイルも再抽出します。コメントや accessor 属性の先読みは維持します。 + +C# の静的 lambda の除外判定は、対象名をまだ含み得る arrow と、現在の宣言区間内の +明示的な戻り値型 prefix に限定します。括弧で囲まれた tuple group は維持します。 +密な1行で前のメンバーまで繰り返し走査・文字列生成する処理を避け、static constructor・ +generic method・型付き／tuple／関数ポインター型 lambda の判定を保ちます。 +Razor・Blazor・CSHTML も共通です。同一行の static method 64件のウォームアップ後の +抽出は、この制限の前で406.5 MB、後で0.93 MBを割り当てました。回帰fixtureには +正例の対照も加え、net8/net9とも上限2 MiBを設けます。別の入力に対する所要時間の +保証ではありません。 + +確認済みメソッド prefix の照合でも、regex 実行の前に開き括弧を必須とします。 +`(` を含まず `) {` で終わる引数の継続断片は照合できないため、先に除外します。 +全シンボル項目の一致と regex 試行回数で検証し、時間の閾値には依存しません。 + +このリポジトリの `0d39d0658` 時点の1,554ファイルを固定し、macOS ARM64・Release +.NET 8・`--parallelism 2 --memory-trace` で空DBから計測したところ、3つの変更前は +76.0秒、変更後は68.9秒でした。managed の総割り当て量は18.46 GBから7.83 GBに +減りました。毎回新しいDBと同じソースを使った単回の観測値であり、一般的な速度を +保証するものではありません。既存シンボルの識別情報・抽出メタデータを維持し、 +欠落していた constructor 7件を回復して、宣言位置の参照7行を除去し、対応する +overload の解決候補も更新しました。 +両方とも全1,554ファイルを警告・抽出エラーなしで完了しています。 diff --git a/src/CodeIndex/Database/DbWriter.AuthoritativeFreshReferenceSourceLookup.cs b/src/CodeIndex/Database/DbWriter.AuthoritativeFreshReferenceSourceLookup.cs index 8958830f0..2c50d30ea 100644 --- a/src/CodeIndex/Database/DbWriter.AuthoritativeFreshReferenceSourceLookup.cs +++ b/src/CodeIndex/Database/DbWriter.AuthoritativeFreshReferenceSourceLookup.cs @@ -21,15 +21,24 @@ end_line INTEGER ) WITHOUT ROWID; CREATE INDEX IF NOT EXISTS temp.idx_authoritative_fresh_source_name_folded - ON {AuthoritativeFreshReferenceSourceSymbolsTableName}(file_id, name_folded) + ON {AuthoritativeFreshReferenceSourceSymbolsTableName}( + file_id, name_folded, + COALESCE(end_line, line) - COALESCE(start_line, line), + COALESCE(start_line, line) DESC, symbol_id, COALESCE(end_line, line)) WHERE name_folded IS NOT NULL; CREATE INDEX IF NOT EXISTS temp.idx_authoritative_fresh_source_display_name_folded - ON {AuthoritativeFreshReferenceSourceSymbolsTableName}(file_id, display_name_folded) + ON {AuthoritativeFreshReferenceSourceSymbolsTableName}( + file_id, display_name_folded, + COALESCE(end_line, line) - COALESCE(start_line, line), + COALESCE(start_line, line) DESC, symbol_id, COALESCE(end_line, line)) WHERE display_name_folded IS NOT NULL; CREATE INDEX IF NOT EXISTS temp.idx_authoritative_fresh_source_name_nocase - ON {AuthoritativeFreshReferenceSourceSymbolsTableName}(file_id, name COLLATE NOCASE) + ON {AuthoritativeFreshReferenceSourceSymbolsTableName}( + file_id, name COLLATE NOCASE, + COALESCE(end_line, line) - COALESCE(start_line, line), + COALESCE(start_line, line) DESC, symbol_id, COALESCE(end_line, line)) WHERE name_folded IS NULL; DELETE FROM temp.{AuthoritativeFreshReferenceSourceSymbolsTableName}; @@ -66,59 +75,57 @@ internal static string PopulateAuthoritativeFreshReferenceSourceLookupSqlForTest private static string BuildMaterializedFreshReferenceSourceSymbolValueSql( string referenceAlias) - // Only the first ranked symbol ID is observed. A symbol matching multiple - // name indexes has the same range and ID in every arm, so duplicates cannot - // change that winner. UNION ALL avoids a distinct temporary B-tree per reference. - // 同一symbolの重複は順位もIDも同じ。先頭1件の選択に不要な参照ごとの重複除去を省く。 + // Each name index supplies its best containing symbol in rank order. The + // final sort compares at most three rows, even for a large overload family. + // Duplicate matches retain the same rank and ID and cannot change the winner. + // 各名前indexから包含順位の先頭だけを取り、最後のsortを最大3行に制限する。 => $""" ( SELECT candidate.symbol_id FROM ( - SELECT source.symbol_id, - source.line, - source.start_line, - source.end_line - FROM temp.{AuthoritativeFreshReferenceSourceSymbolsTableName} AS source - WHERE {referenceAlias}.container_name IS NOT NULL - AND {referenceAlias}.container_name <> '' - AND source.file_id = {referenceAlias}.file_id - AND source.name_folded = {referenceAlias}.container_name_folded + {BuildRankedFreshReferenceSourceProbeSql(referenceAlias, + $"source.name_folded = {referenceAlias}.container_name_folded")} UNION ALL - SELECT source.symbol_id, - source.line, - source.start_line, - source.end_line - FROM temp.{AuthoritativeFreshReferenceSourceSymbolsTableName} AS source - WHERE {referenceAlias}.container_name IS NOT NULL - AND {referenceAlias}.container_name <> '' - AND source.file_id = {referenceAlias}.file_id - AND source.display_name_folded = {referenceAlias}.container_name_folded + {BuildRankedFreshReferenceSourceProbeSql(referenceAlias, + $"source.display_name_folded = {referenceAlias}.container_name_folded")} UNION ALL - SELECT source.symbol_id, - source.line, - source.start_line, - source.end_line - FROM temp.{AuthoritativeFreshReferenceSourceSymbolsTableName} AS source - WHERE {referenceAlias}.container_name IS NOT NULL - AND {referenceAlias}.container_name <> '' - AND source.file_id = {referenceAlias}.file_id - AND source.name_folded IS NULL - AND source.name = {referenceAlias}.container_name COLLATE NOCASE + {BuildRankedFreshReferenceSourceProbeSql(referenceAlias, + $"source.name_folded IS NULL AND source.name = {referenceAlias}.container_name COLLATE NOCASE")} ) AS candidate - WHERE {referenceAlias}.line BETWEEN COALESCE(candidate.start_line, candidate.line) - AND COALESCE(candidate.end_line, candidate.line) - ORDER BY (COALESCE(candidate.end_line, candidate.line) - - COALESCE(candidate.start_line, candidate.line)), - COALESCE(candidate.start_line, candidate.line) DESC, + ORDER BY candidate.range_width, + candidate.start_line DESC, candidate.symbol_id LIMIT 1 ) """; + private static string BuildRankedFreshReferenceSourceProbeSql( + string referenceAlias, + string namePredicate) + => $""" + SELECT * FROM ( + SELECT source.symbol_id, + COALESCE(source.end_line, source.line) - + COALESCE(source.start_line, source.line) AS range_width, + COALESCE(source.start_line, source.line) AS start_line + FROM temp.{AuthoritativeFreshReferenceSourceSymbolsTableName} AS source + WHERE {referenceAlias}.container_name IS NOT NULL + AND {referenceAlias}.container_name <> '' + AND source.file_id = {referenceAlias}.file_id + AND {namePredicate} + AND {referenceAlias}.line BETWEEN COALESCE(source.start_line, source.line) + AND COALESCE(source.end_line, source.line) + ORDER BY COALESCE(source.end_line, source.line) - COALESCE(source.start_line, source.line), + COALESCE(source.start_line, source.line) DESC, + source.symbol_id + LIMIT 1 + ) + """; + internal static string BuildMaterializedFreshReferenceSourceSymbolValueSqlForTesting( string referenceAlias) => BuildMaterializedFreshReferenceSourceSymbolValueSql(referenceAlias); diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs index 5237b6192..4ef237998 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs @@ -1670,7 +1670,8 @@ private static CSharpPropertyMatchCandidate BuildCSharpPropertyMatchLine( : (int?)null; if (HasCSharpTopLevelFieldInitializer(matchLine) - || openBraceLineIndex >= 0 && IsCSharpConfirmedMemberOrMethodPrefix(matchLine)) + || openBraceLineIndex >= 0 && IsCSharpConfirmedMemberOrMethodPrefix( + matchLine, applyCSharpRegexProbeOptimizations, csharpRegexProbeCounts)) { return ContinueConfirmedCSharpPropertyMatch( lines, @@ -1745,7 +1746,8 @@ private static CSharpPropertyMatchCandidate BuildCSharpPropertyMatchLine( } if (HasCSharpTopLevelFieldInitializer(normalizedCombined) - || openBraceLineIndex >= 0 && IsCSharpConfirmedMemberOrMethodPrefix(normalizedCombined)) + || openBraceLineIndex >= 0 && IsCSharpConfirmedMemberOrMethodPrefix( + normalizedCombined, applyCSharpRegexProbeOptimizations, csharpRegexProbeCounts)) { return ContinueConfirmedCSharpPropertyMatch( lines, @@ -1814,6 +1816,7 @@ private static CSharpPropertyMatchCandidate ContinueConfirmedCSharpPropertyMatch StringBuilder? accessorProbeBuilder = null; var accessorProbeStatus = CSharpAccessorProbeStatus.Rejected; + CSharpPropertyMatchCandidate? confirmedMethodHeader = null; if (openBraceLineIndex >= 0 && openBraceExclusiveEndColumn.HasValue) { accessorProbeBuilder = BuildCSharpAccessorProbeBuilder( @@ -1831,10 +1834,12 @@ private static CSharpPropertyMatchCandidate ContinueConfirmedCSharpPropertyMatch openBraceExclusiveEndColumn); } - if (accessorProbeStatus == CSharpAccessorProbeStatus.Rejected - && CSharpConfirmedMethodPrefixRegex.IsMatch(normalizedCombined)) + if (CSharpConfirmedMethodPrefixRegex.IsMatch(normalizedCombined)) { - return new CSharpPropertyMatchCandidate(normalizedCombined, currentLineIndex, currentLineIndex); + confirmedMethodHeader = new CSharpPropertyMatchCandidate( + normalizedCombined, currentLineIndex, currentLineIndex); + if (accessorProbeStatus == CSharpAccessorProbeStatus.Rejected) + return confirmedMethodHeader.Value; } } @@ -1857,13 +1862,12 @@ private static CSharpPropertyMatchCandidate ContinueConfirmedCSharpPropertyMatch openBraceExclusiveEndColumn.Value, i); accessorProbeStatus = ClassifyCSharpAccessorProbe(accessorProbeBuilder.ToString()); - if (accessorProbeStatus == CSharpAccessorProbeStatus.Rejected - && CSharpConfirmedMethodPrefixRegex.IsMatch(CollapseCSharpGenericTypeWhitespace(builder.ToString()))) + var header = CollapseCSharpGenericTypeWhitespace(builder.ToString()); + if (CSharpConfirmedMethodPrefixRegex.IsMatch(header)) { - return new CSharpPropertyMatchCandidate( - CollapseCSharpGenericTypeWhitespace(builder.ToString()), - i, - i); + confirmedMethodHeader = new CSharpPropertyMatchCandidate(header, i, i); + if (accessorProbeStatus == CSharpAccessorProbeStatus.Rejected) + return confirmedMethodHeader.Value; } } else if (accessorProbeBuilder != null @@ -1873,6 +1877,17 @@ private static CSharpPropertyMatchCandidate ContinueConfirmedCSharpPropertyMatch accessorProbeStatus = ClassifyCSharpAccessorProbe(accessorProbeBuilder.ToString()); } + // A brace on its own line leaves the accessor probe pending. Once a + // later body token rejects it, use the already confirmed method header + // instead of merging the method and every following body up to a field. + // 単独の開きbraceではaccessor判定が保留になる。後続tokenで棄却されたら + // 確認済みmethod headerを返し、後続bodyまで連結し続けない。 + if (accessorProbeStatus == CSharpAccessorProbeStatus.Rejected + && confirmedMethodHeader.HasValue) + { + return confirmedMethodHeader.Value; + } + if (accessorProbeStatus == CSharpAccessorProbeStatus.Found) { return new CSharpPropertyMatchCandidate( @@ -1894,7 +1909,10 @@ private static CSharpPropertyMatchCandidate ContinueConfirmedCSharpPropertyMatch return new CSharpPropertyMatchCandidate(normalizedCombined, currentLineIndex, currentLineIndex); } - private static bool IsCSharpConfirmedMemberOrMethodPrefix(string line) + private static bool IsCSharpConfirmedMemberOrMethodPrefix( + string line, + bool applyCSharpRegexProbeOptimizations, + CSharpRegexProbeCounts? csharpRegexProbeCounts) { var openBraceIndex = line.IndexOf('{'); if (openBraceIndex < 0) @@ -1904,9 +1922,25 @@ private static bool IsCSharpConfirmedMemberOrMethodPrefix(string line) while (prefixEnd >= 0 && char.IsWhiteSpace(line[prefixEnd])) prefixEnd--; - return prefixEnd >= 0 && line[prefixEnd] == ')' - ? CSharpConfirmedMethodPrefixRegex.IsMatch(line) - : CSharpConfirmedMemberPrefixRegex.IsMatch(line); + if (prefixEnd >= 0 && line[prefixEnd] == ')') + { + // Parameter continuation fragments can end with `) {` without ever + // opening a parameter list. The method regex requires a literal `(`. + // 引数の継続断片は `(` を含まず `) {` で終わり得るが、method regex は + // 必ず `(` を消費するため、成立しない高コストの照合を省略できる。 + if (applyCSharpRegexProbeOptimizations && line.IndexOf('(') < 0) + { + if (csharpRegexProbeCounts != null) + csharpRegexProbeCounts.MethodConfirmationLiteralSkipCount++; + return false; + } + + if (csharpRegexProbeCounts != null) + csharpRegexProbeCounts.MethodConfirmationRegexAttemptCount++; + return CSharpConfirmedMethodPrefixRegex.IsMatch(line); + } + + return CSharpConfirmedMemberPrefixRegex.IsMatch(line); } // Prefer the raw line's `{` column (to preserve original positioning for body slicing), @@ -2439,7 +2473,11 @@ private static bool IsCSharpStaticLambdaHeaderCandidate( string? enclosingTypeName) { var boundedNameIndex = Math.Min(Math.Max(0, nameIndex), matchLine.Length); - var searchIndex = 0; + // A header ends just after its arrow, so an earlier arrow cannot contain + // this name. Keep the second arrow character for the boundary case. + // header は arrow 直後で終わるため、名前より前で終わる arrow は対象外。 + // 境界条件のため arrow の2文字目だけは検索範囲に残す。 + var searchIndex = Math.Max(0, boundedNameIndex - 1); while (searchIndex < matchLine.Length) { var arrowIndex = matchLine.IndexOf("=>", searchIndex, StringComparison.Ordinal); @@ -2449,6 +2487,7 @@ private static bool IsCSharpStaticLambdaHeaderCandidate( if (TryGetCSharpStaticLambdaHeaderRange( matchLine, arrowIndex, + boundedNameIndex, enclosingTypeName, out var headerStart, out var headerEnd) @@ -2467,6 +2506,7 @@ private static bool IsCSharpStaticLambdaHeaderCandidate( private static bool TryGetCSharpStaticLambdaHeaderRange( string matchLine, int arrowIndex, + int nameIndex, string? enclosingTypeName, out int headerStart, out int headerEnd) @@ -2527,10 +2567,21 @@ private static bool TryGetCSharpStaticLambdaHeaderRange( return true; } - if (!hasParenthesizedParameters + if (!hasParenthesizedParameters) + return false; + + // Outside tuple groups, return types and declaration headers cannot cross + // semicolons or braces. + // Bound explicit-return probes before allocating or matching prefixes from + // earlier declarations, and skip later headers that cannot contain the name. + // tuple group外のセミコロン・braceは戻り値型・宣言headerを分断する。prefix生成と + // regex照合を避け、対象名を含めない後続headerも除外する。 + var declarationBoundary = FindCSharpStaticLambdaDeclarationBoundary(matchLine, parameterStart); + if (nameIndex <= declarationBoundary || !TryFindCSharpExplicitReturnStaticLambdaModifier( matchLine, parameterStart, + declarationBoundary + 1, enclosingTypeName, out modifierStart)) { @@ -2542,6 +2593,23 @@ private static bool TryGetCSharpStaticLambdaHeaderRange( return true; } + private static int FindCSharpStaticLambdaDeclarationBoundary(string text, int parameterStart) + { + var parentheses = 0; + for (var index = parameterStart - 1; index >= 0; index--) + { + var ch = text[index]; + if (ch == ')') + parentheses++; + else if (ch == '(' && parentheses > 0) + parentheses--; + else if (parentheses == 0 && ch is ';' or '{' or '}') + return index; + } + + return -1; + } + private static bool TryReadCSharpStaticLambdaModifiersBackward( string text, int cursor, @@ -2578,6 +2646,7 @@ private static bool TryReadCSharpStaticLambdaModifiersBackward( private static bool TryFindCSharpExplicitReturnStaticLambdaModifier( string text, int parameterStart, + int minimumModifierStart, string? enclosingTypeName, out int modifierStart) { @@ -2589,7 +2658,7 @@ private static bool TryFindCSharpExplicitReturnStaticLambdaModifier( "static", searchBefore - 1, StringComparison.Ordinal); - if (staticIndex < 0) + if (staticIndex < minimumModifierStart) return false; searchBefore = staticIndex; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Contracts.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Contracts.cs index 5156231c8..8829c3a35 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Contracts.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Contracts.cs @@ -22,7 +22,11 @@ public static partial class SymbolExtractor // and their reference-container boundaries are persisted for existing indexes. // バージョン 18 (#5228) は既存 index の C# symbol を再抽出し、セミコロン形式 record の // body range と参照 container 境界を永続化する。 - public const int CSharpContractVersion = 18; + // Version 19 bounds confirmed method lookahead and recovers constructor headers + // previously swallowed with subsequent method bodies before a trailing field. + // バージョン 19 は確認済み method の先読みを止め、後続 body と一緒に + // 末尾 field まで飲み込まれていた constructor header を再抽出する。 + public const int CSharpContractVersion = 19; public const int DockerfileContractVersion = 2; public const int MakefileContractVersion = 2; public const int StyleAndXamlContractVersion = 2; @@ -80,7 +84,7 @@ public static int GetContractVersion(string? lang) { null or "" => DefaultContractVersion, "python" => PythonContractVersion, - "csharp" => CSharpContractVersion, + "csharp" or "razor" or "blazor" or "cshtml" => CSharpContractVersion, "dockerfile" => DockerfileContractVersion, "makefile" => MakefileContractVersion, "sass" or "stylus" => StyleAndXamlContractVersion, diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 61f1592c7..0099b382f 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -355,6 +355,8 @@ private sealed class CSharpRegexProbeCounts public int PropertyStructuralShapeSkipCount { get; set; } public int PropertyHeaderRegexAttemptCount { get; set; } public int MethodHeaderRegexAttemptCount { get; set; } + public int MethodConfirmationRegexAttemptCount { get; set; } + public int MethodConfirmationLiteralSkipCount { get; set; } public int PlainFieldTerminatorSkipCount { get; set; } public int PlainFieldRegexAttemptCount { get; set; } public int RecoverablePlainFieldTerminatorSkipCount { get; set; } @@ -374,6 +376,8 @@ internal readonly record struct CSharpRegexProbeMetrics( int PropertyStructuralShapeSkipCount, int PropertyHeaderRegexAttemptCount, int MethodHeaderRegexAttemptCount, + int MethodConfirmationRegexAttemptCount, + int MethodConfirmationLiteralSkipCount, int PlainFieldTerminatorSkipCount, int PlainFieldRegexAttemptCount, int RecoverablePlainFieldTerminatorSkipCount, @@ -456,6 +460,8 @@ internal static List ExtractForCSharpRegexProbeTesting( counts.PropertyStructuralShapeSkipCount, counts.PropertyHeaderRegexAttemptCount, counts.MethodHeaderRegexAttemptCount, + counts.MethodConfirmationRegexAttemptCount, + counts.MethodConfirmationLiteralSkipCount, counts.PlainFieldTerminatorSkipCount, counts.PlainFieldRegexAttemptCount, counts.RecoverablePlainFieldTerminatorSkipCount, diff --git a/tests/CodeIndex.Tests/AuthoritativeFreshRawBulkInsertTests.cs b/tests/CodeIndex.Tests/AuthoritativeFreshRawBulkInsertTests.cs index e81f45681..d2c370dd6 100644 --- a/tests/CodeIndex.Tests/AuthoritativeFreshRawBulkInsertTests.cs +++ b/tests/CodeIndex.Tests/AuthoritativeFreshRawBulkInsertTests.cs @@ -426,6 +426,8 @@ public void ChunkTextBindings_ReuseScratchSpaceForLongUtf8Values() [InlineData("go", "go")] [InlineData("rust", "rs")] [InlineData("cpp", "cpp")] + [InlineData("kotlin", "kt")] + [InlineData("vb", "vb")] public void ReferenceSourceLookup_PreservesMultiFileFoldFallbackAndNestedRankingAcrossBatches( string language, string extension) @@ -437,6 +439,7 @@ public void ReferenceSourceLookup_PreservesMultiFileFoldFallbackAndNestedRanking long duplicateProbeSourceId; long legacyAsciiSourceId; long secondNestedSourceId; + long crossArmSourceId; using (var graph = _writer.BeginReferenceGraphRefreshScope( forceFullRefresh: true, @@ -470,6 +473,9 @@ public void ReferenceSourceLookup_PreservesMultiFileFoldFallbackAndNestedRanking SourceSymbol(firstFileId, "ÅLegacy", line: 76, startLine: 76, endLine: 85), SourceSymbol(secondFileId, "Caller", line: 1, startLine: 1, endLine: 100), SourceSymbol(secondFileId, "Caller", line: 50, startLine: 50, endLine: 60), + SourceSymbol(firstFileId, "Caller", line: 1, startLine: 1, endLine: 2), + SourceSymbol(firstFileId, "AliasOwner", line: 16, startLine: 16, endLine: 18, + displayNameFolded: "caller"), ]); Execute($""" UPDATE symbols @@ -503,6 +509,7 @@ SELECT id FROM symbols AND name = 'Caller' AND start_line = 50 """); + crossArmSourceId = ScalarLong("SELECT id FROM symbols WHERE name = 'AliasOwner'"); var references = Enumerable.Range(0, 40) .Select(index => SourceReference( @@ -546,6 +553,8 @@ SELECT id FROM symbols "second_file_probe", line: 55, containerName: "Caller")); + references.Add(SourceReference(firstFileId, "cross_arm_rank_probe", line: 18, containerName: "Caller")); + references.Add(SourceReference(firstFileId, "outside_all_ranges_probe", line: 101, containerName: "Caller")); _writer.InsertReferencesForNewFilesInAtomicFileScope( references, @@ -559,7 +568,7 @@ SELECT COUNT(DISTINCT file_id) FROM temp.{DbWriter.AuthoritativeFreshReferenceSourceSymbolsTableName} """)); Assert.Equal( - 9L, + 11L, ScalarLong($""" SELECT COUNT(*) FROM temp.{DbWriter.AuthoritativeFreshReferenceSourceSymbolsTableName} @@ -592,6 +601,8 @@ WHERE symbol_name LIKE 'nested_probe_%' Assert.Null(SourceSymbolId("null_container_probe")); Assert.Null(SourceSymbolId("empty_container_probe")); Assert.Equal(secondNestedSourceId, SourceSymbolId("second_file_probe")); + Assert.Equal(crossArmSourceId, SourceSymbolId("cross_arm_rank_probe")); + Assert.Null(SourceSymbolId("outside_all_ranges_probe")); const string sourceSnapshotSql = """ SELECT group_concat(COALESCE(source_symbol_id, 'null'), '|') @@ -688,6 +699,8 @@ FROM reference_row AS r Assert.DoesNotContain( sourceLookupPlan, detail => detail.Contains("UNION USING TEMP B-TREE", StringComparison.OrdinalIgnoreCase)); + Assert.Single(sourceLookupPlan.Where(detail => detail.Contains( + "USE TEMP B-TREE FOR ORDER BY", StringComparison.OrdinalIgnoreCase))); raw.Complete(); transaction.Commit(); @@ -707,6 +720,70 @@ IReadOnlyList ExplainQueryPlan( } } + [Fact] + public void ReferenceSourceLookup_RankedOverloadsAvoidSortingEveryMatchingSymbol() + { + using var graph = _writer.BeginReferenceGraphRefreshScope( + forceFullRefresh: true, + useFreshReferenceResolutionDefaults: true); + using var transaction = _writer.BeginTransaction(); + using var raw = _writer.BeginAuthoritativeFreshBulkInsertScope( + enabled: true, + CancellationToken.None)!; + var fileId = InsertNewFile("src/ranked-overloads.cs"); + _writer.InsertSymbols(Enumerable.Range(1, 128).Select(index => new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = "Caller", + DisplayNameFolded = "caller", + Line = index, + StartLine = index, + EndLine = 257 - index, + }).ToArray()); + raw.MaterializeReferenceSourceSymbols([new ReferenceRecord + { + FileId = fileId, + SymbolName = "Target", + ReferenceKind = "call", + ContainerName = "Caller", + Line = 128, + }]); + var expected = ScalarLong("SELECT id FROM symbols WHERE start_line = 128"); + var lookup = DbWriter.BuildMaterializedFreshReferenceSourceSymbolValueSqlForTesting("r"); + using var command = _db.Connection.CreateCommand(); + command.CommandText = $""" + WITH RECURSIVE reference_row(ordinal, file_id, line, container_name, container_name_folded) AS ( + SELECT 1, {fileId}, 128, 'Caller', 'caller' + UNION ALL + SELECT ordinal + 1, file_id, line, container_name, container_name_folded + FROM reference_row WHERE ordinal < 64 + ) + SELECT {lookup} FROM reference_row AS r + """; + var callbacks = 0; + SQLitePCL.delegate_progress progress = _ => { callbacks++; return 0; }; + SQLitePCL.raw.sqlite3_progress_handler(_db.Connection.Handle, 1000, progress, null!); + try + { + using var reader = command.ExecuteReader(); + var rows = 0; + while (reader.Read()) + { + Assert.Equal(expected, reader.GetInt64(0)); + rows++; + } + Assert.Equal(64, rows); + } + finally + { + SQLitePCL.raw.sqlite3_progress_handler(_db.Connection.Handle, 0, null!, null!); + } + Assert.InRange(callbacks, 1, 32); + raw.Complete(); + transaction.Commit(); + } + [Fact] public void ReferenceSourceLookup_FileSavepointRollbackRestoresPreviousRowsAndReprepares() { diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 5d91150cc..42130dd71 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -8603,6 +8603,37 @@ public void GetUnchangedFileId_ReturnsNullWhenLanguageExtractorVersionIsStale() Assert.Null(id); } + [Theory] + [InlineData("csharp", 18)] + [InlineData("razor", 1)] + [InlineData("blazor", 1)] + [InlineData("cshtml", 1)] + public void GetUnchangedFileId_InvalidatesPriorCSharpLookaheadContracts(string language, int previousVersion) + { + var modified = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var file = new FileRecord + { + Path = $"src/Counter.{language}", + Lang = language, + Size = 50, + Lines = 5, + Modified = modified, + }; + _writer.UpsertFile(file); + _writer.SetMeta( + DbContext.GetSymbolExtractorVersionMetaKey(language), + previousVersion.ToString(CultureInfo.InvariantCulture)); + + Assert.Equal(SymbolExtractor.CSharpContractVersion, SymbolExtractor.GetContractVersion(language)); + Assert.True(SymbolExtractor.GetContractVersion(language) > previousVersion); + Assert.Null(_writer.GetUnchangedFileId(file.Path, modified, language: language)); + Assert.Null(_writer.GetUnchangedFileIdByStat(file.Path, modified, file.Size, language: language)); + + _writer.StampSymbolExtractorVersions([language]); + Assert.NotNull(_writer.GetUnchangedFileId(file.Path, modified, language: language)); + Assert.NotNull(_writer.GetUnchangedFileIdByStat(file.Path, modified, file.Size, language: language)); + } + [Fact] public void GetUnchangedFileId_InvalidatesPriorMarkdownHeadingRangeContract_Issue4910() { diff --git a/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs b/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs index aaf0c0d5f..178bde8a4 100644 --- a/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs +++ b/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs @@ -191,7 +191,7 @@ AND source_symbol_id IS NULL Assert.Equal(2, CountOccurrences(materializedFreshSql, "UNION")); Assert.Equal(2, CountOccurrences(materializedFreshSql, "UNION ALL")); Assert.Contains( - "COALESCE(candidate.start_line, candidate.line) DESC", + "candidate.start_line DESC", materializedFreshSql, StringComparison.Ordinal); Assert.Equal(28, CountOccurrences(materializedFreshSql, "?")); diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index 9bbdc48d1..4bf50b194 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -6852,7 +6852,7 @@ FROM symbol_references SymbolExtractor.CSharpContractVersion.ToString( System.Globalization.CultureInfo.InvariantCulture), versionCmd.ExecuteScalar() as string); - Assert.Equal(18, SymbolExtractor.CSharpContractVersion); + Assert.Equal(19, SymbolExtractor.CSharpContractVersion); } finally { @@ -7118,7 +7118,7 @@ ORDER BY symbol_name SymbolExtractor.CSharpContractVersion.ToString( System.Globalization.CultureInfo.InvariantCulture), versionCmd.ExecuteScalar() as string); - Assert.Equal(18, SymbolExtractor.CSharpContractVersion); + Assert.Equal(19, SymbolExtractor.CSharpContractVersion); } finally { diff --git a/tests/CodeIndex.Tests/SymbolExtractorCSharpRegexProbeTests.cs b/tests/CodeIndex.Tests/SymbolExtractorCSharpRegexProbeTests.cs index cfd8bf6b0..460fe0926 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorCSharpRegexProbeTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorCSharpRegexProbeTests.cs @@ -311,6 +311,139 @@ public void Run(int value) { MeasureRepresentativeAllocatedBytes(content)); } + [Theory] + [InlineData("csharp")] + [InlineData("razor")] + [InlineData("blazor")] + [InlineData("cshtml")] + public void Extract_CSharpConfirmedMethodLookahead_StopsAfterAccessorRejection(string language) + { + var methods = string.Join('\n', Enumerable.Range(0, 32).Select(index => $$""" + static int CountOccurrences{{index}}(string value, string search) + { + // Delay the first meaningful body token. + + return value.Length + search.Length; + } + """)); + var content = $$""" + class Counter + { + public Counter() + : this(0) + { + } + + public Counter(int seed) + : base() + { + Sentinel = seed; + } + + {{methods}} + int Sentinel; + int Value + { + [System.Obsolete] + get; + } + } + """; + SymbolExtractor.Extract(1, language, content); + var before = GC.GetAllocatedBytesForCurrentThread(); + var symbols = SymbolExtractor.Extract(1, language, content); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + Assert.Equal(34, symbols.Count(symbol => symbol.Kind == "function")); + var constructors = symbols.Where(symbol => symbol.Name == "Counter" && symbol.Kind == "function").ToArray(); + Assert.Equal(2, constructors.Length); + Assert.All(constructors, symbol => Assert.Equal(symbol.StartLine + 2, symbol.BodyStartLine)); + Assert.Contains(constructors, symbol => symbol.Signature!.Contains(": this(0)")); + Assert.Contains(constructors, symbol => symbol.Signature!.Contains(": base()")); + Assert.Contains(symbols, symbol => symbol.Name == "Sentinel" && symbol.Kind == "field"); + Assert.Contains(symbols, symbol => symbol.Name == "Value" && symbol.Kind == "property"); + Assert.All(symbols.Where(symbol => symbol.Name.StartsWith("CountOccurrences", StringComparison.Ordinal)), symbol => + { + Assert.Equal(symbol.StartLine + 5, symbol.EndLine); + Assert.Equal(symbol.StartLine + 1, symbol.BodyStartLine); + Assert.DoesNotContain("return", symbol.Signature!); + }); + Assert.InRange(allocated, 0L, 512 * 1024L); + } + + [Fact] + public void Extract_CSharpMethodConfirmation_RequiresOpeningParenthesis() + { + const string content = """ + class Example + { + public int Sum( + LongDeclarationParameterType left, + LongDeclarationParameterType right) + { + return 1; + } + public int Value { get; } + } + """; + var baseline = Extract(content, applyOptimizations: false, out var baselineMetrics); + var optimized = Extract(content, applyOptimizations: true, out var optimizedMetrics); + + AssertSymbolsEqual(baseline, optimized); + Assert.Contains(optimized, symbol => symbol.Kind == "function" && symbol.Name == "Sum"); + Assert.Contains(optimized, symbol => symbol.Kind == "property" && symbol.Name == "Value"); + Assert.Equal(0, baselineMetrics.MethodConfirmationLiteralSkipCount); + Assert.True(optimizedMetrics.MethodConfirmationLiteralSkipCount > 0); + Assert.True(optimizedMetrics.MethodConfirmationRegexAttemptCount > 0); + Assert.True(optimizedMetrics.MethodConfirmationRegexAttemptCount < baselineMetrics.MethodConfirmationRegexAttemptCount); + } + + [Theory] + [InlineData("csharp")] + [InlineData("razor")] + [InlineData("blazor")] + [InlineData("cshtml")] + public void Extract_CSharpStaticLambdaGate_BoundsRepeatedSameLineDeclarations(string language) + { + var methods = string.Join(' ', Enumerable.Range(0, 64) + .Select(index => $"public static int Method{index}() => {index};")); + var content = $$""" + class Example { {{methods}} } + unsafe class Controls + { + static Controls() => Initialize(); + public static void Initialize() { } + public static T Identity(T value) => value; + void Callbacks() + { + var typed = static int (int value) => value; + var tuple = static (int Left, int Right) (int value) => (value, value); + var pointer = static delegate* () => null; + var inferred = static value => value; + } + } + """; + SymbolExtractor.Extract(1, language, content); + var before = GC.GetAllocatedBytesForCurrentThread(); + var symbols = SymbolExtractor.Extract(1, language, content); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + var functions = symbols.Where(symbol => symbol.Kind == "function").ToArray(); + Assert.Equal(68, functions.Length); + for (var index = 0; index < 64; index++) + { + var method = Assert.Single(functions.Where(symbol => symbol.Name == $"Method{index}")); + Assert.Equal("Example", method.ContainerName); + Assert.Equal("int", method.ReturnType); + Assert.Equal((1, 1, 1, 1), (method.StartLine, method.EndLine, method.BodyStartLine, method.BodyEndLine)); + Assert.Equal(content.IndexOf($"public static int Method{index}(", StringComparison.Ordinal), method.StartColumn); + } + Assert.Single(functions.Where(symbol => symbol.Name == "Controls")); + Assert.Single(functions.Where(symbol => symbol.Name == "Initialize")); + Assert.Single(functions.Where(symbol => symbol.Name == "Identity")); + Assert.Single(functions.Where(symbol => symbol.Name == "Callbacks")); + Assert.InRange(allocated, 0L, 2 * 1024 * 1024L); + } + [Fact] public void AllocationRegressionComparison_UsesMedianNoiseAllowanceAndRejectsMaterialIncrease_Issue5244() {