From 3e8e75d58ecfec46e3a83f85778c7b72191628f8 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 12 Sep 2026 23:00:31 +0900 Subject: [PATCH 1/2] Resume bounded C# origin classification (#5348) --- DEVELOPER_GUIDE.md | 8 +- TESTING_GUIDE.md | 4 + USER_GUIDE.md | 4 +- changelog.d/unreleased/5348.fixed.md | 17 + docs/find-scan-controls.md | 54 +++ src/CodeIndex/Cli/CliFlagSchema.cs | 1 + src/CodeIndex/Cli/ConsoleUi.cs | 2 +- ...yCommandRunner.ArgumentParser.Locations.cs | 11 + .../Cli/QueryCommandRunner.ArgumentParser.cs | 2 + .../Cli/QueryCommandRunner.AuditAll.cs | 2 + src/CodeIndex/Cli/QueryCommandRunner.Batch.cs | 3 +- .../Cli/QueryCommandRunner.Database.cs | 1 + src/CodeIndex/Cli/QueryCommandRunner.Find.cs | 23 +- .../Cli/QueryCommandRunner.ResultEnvelopes.cs | 2 + .../Cli/QueryCommandRunner.SearchRecipes.cs | 6 + src/CodeIndex/Cli/QueryCommandRunner.cs | 1 + .../Database/DbSearchReader.CSharpOrigins.cs | 100 +++++ src/CodeIndex/Database/DbSearchReader.cs | 46 +- src/CodeIndex/Database/IndexedFindModels.cs | 8 +- src/CodeIndex/Database/IndexedFindPipeline.cs | 6 + .../Database/SearchMatchClassifier.CSharp.cs | 410 ++++++++++-------- src/CodeIndex/Models/QueryResults.cs | 9 +- .../OriginContinuationIssue5348Tests.cs | 239 ++++++++++ .../QueryCommandRunnerSearchIssue5307Tests.cs | 5 + 24 files changed, 726 insertions(+), 238 deletions(-) create mode 100644 changelog.d/unreleased/5348.fixed.md create mode 100644 src/CodeIndex/Database/DbSearchReader.CSharpOrigins.cs create mode 100644 tests/CodeIndex.Tests/OriginContinuationIssue5348Tests.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index ad6ecef0f..ebe8c2bef 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -58,11 +58,13 @@ untruncated output. Attestation still relies on the verifier's exit status. ## C# search-origin context +Issue #5348 adds explicit `--origin-passes` (1–16, default 1) to CLI search/audit/regex-find. `DbSearchReader.CSharpOrigins.cs` supplies successive bounded windows to one `CSharpOriginContext`; lexical locals, interpolation frames, labels and schema state survive each window. Limits apply per pass, including overlapping chunk characters, with at most 65,536 lines / 128 Mi source characters retained per file. State never survives a query; local `total_changes()` and external `data_version` checks discard contexts if their indexed generation changes. Preserve missing/conflicting-line holes, cancellation, fixed per-query pass budgets, cursor/replay binding, and bounded retry diagnostics. A zero-progress window cannot request another pass. See [continuation behavior](docs/find-scan-controls.md#bounded-c-lexical-continuation-5348). + Read the bounded prefix independently of the last returned chunk so later closing delimiters cannot change earlier origins across pages. Interpolation frames isolate schema-call state (including alignment commas) while retaining nested builder detection. Ordinary interpolation format text cannot cross a physical newline; verbatim/raw formats can. Issue #5321 resumes ordinary/verbatim/raw interpolation with at most 64 active interpolation frames and 64 balanced delimiters per expression. Nested comments/strings are consumed by the same cancellable prefix pass. Merge adjacent literal spans, preserve UTF-16 coordinates, and keep unsupported formats or unbalanced/missing context unknown with bounded `SearchMatchFacet.OriginUnavailable` reason/start/extent evidence. No work budget depends on requested matches or pagination. -`DbSearchReader.AttachCSharpOriginLines` supplies shared, indexed file prefixes to the snippet classifier. Preserve the per-file 4,096-line, 8 Mi-character and 128-chunk read limits, including overlap accounting; query pagination must not change those budgets. The character limit leaves room above existing 4 Mi-character semantic-analysis windows. Keep missing lines absent so the lexical classifier returns `unknown` rather than assuming code. Ordinary/token-boundary row and count paths and MCP must retain identical origin decisions and original UTF-16 coordinates. No persisted schema changes are involved. +`DbSearchReader.AttachCSharpOriginLines` supplies shared, indexed file prefixes to the snippet classifier. Preserve the per-pass, per-file 4,096-line, 8 Mi-character and 128-chunk read limits, including overlap accounting; query pagination must not change those budgets. The character limit leaves room above existing 4 Mi-character semantic-analysis windows. Keep missing lines absent so the lexical classifier returns `unknown` rather than assuming code. Ordinary/token-boundary row and count paths and MCP must retain identical origin decisions and original UTF-16 coordinates. No persisted schema changes are involved. Build `CSharpOriginContext` once per file prefix with cancellation and share its origin spans across rows and occurrences. Track schema argument positions during that same lexical pass, with at most 64 active builder invocations and a 64-line lookback; overflow leaves affected labels unknown. Cache regex/help classification per opening line, with cancellation at label lookup. Do not rescan preceding lines or reconstruct schema context per match or per literal. @@ -4623,11 +4625,13 @@ JSON の `installer_output_incomplete` に記録します。判明した親の ## C# 検索 origin のコンテキスト +Issue #5348 は CLI の search／audit／regex-find に明示的な `--origin-passes`（1〜16、既定 1）を追加します。`DbSearchReader.CSharpOrigins.cs` は同じ `CSharpOriginContext` へ上限付きの窓を順次渡し、字句状態、補間フレーム、ラベル、schema の文脈を引き継ぎます。重複チャンクの文字数を含む上限はパスごとに適用し、保持するソースはファイルごとに最大 65,536 行／128 Mi 文字です。クエリを越えて状態を保持せず、同一接続の `total_changes()` と外部変更の `data_version` によって索引世代の変化時に文脈を破棄します。欠落・不整合な行、キャンセル、クエリごとに固定したパス数、カーソル・再実行条件の紐づけ、上限付き再試行診断を維持してください。前進できない窓では追加パスを案内しません。[継続動作の説明](docs/find-scan-controls.md#上限付き-c-字句分類の継続-5348)も参照してください。 + 返す最後のチャンクとは独立して上限付きの先頭部分を読み取り、後続の閉じ区切りによってページ間で先行箇所の origin が変わらないようにします。補間フレームは配置指定のカンマを含む schema 呼び出し状態を分離し、入れ子の builder 検出も維持します。通常の補間書式部分では物理改行を許可せず、verbatim/raw の書式部分では許可します。 Issue #5321 は、同時に開いている補間フレームを最大 64、式ごとの対応する区切りを最大 64 として、通常／verbatim／raw 補間から走査を再開します。入れ子のコメントと文字列も同じキャンセル可能な先頭部分の走査で処理します。隣接するリテラル区間を結合し、UTF-16 座標を保持してください。未対応の書式や不均衡・欠落した文脈は、上限付きの `SearchMatchFacet.OriginUnavailable` の理由・開始位置・範囲を伴う不明状態にします。処理上限は一致数やページングに依存しません。 -`DbSearchReader.AttachCSharpOriginLines` は共有のインデックス済みファイル先頭部分を snippet 分類器へ渡します。ファイルごとの 4,096 行、8 Mi 文字、128 チャンクの読み取り上限と重複分の計上を維持し、query のページングで上限を変えないでください。文字数上限は既存の 4 Mi 文字の意味解析ウィンドウより大きく設定しています。欠落行を補わず、字句分類器がコードと推測せず `unknown` を返すようにします。通常／token-boundary の行・件数経路と MCP で同じ origin 判定と元の UTF-16 座標を維持してください。永続スキーマの変更はありません。 +`DbSearchReader.AttachCSharpOriginLines` は共有のインデックス済みファイル先頭部分を snippet 分類器へ渡します。ファイル・パスごとの 4,096 行、8 Mi 文字、128 チャンクの読み取り上限と重複分の計上を維持し、query のページングで上限を変えないでください。文字数上限は既存の 4 Mi 文字の意味解析ウィンドウより大きく設定しています。欠落行を補わず、字句分類器がコードと推測せず `unknown` を返すようにします。通常／token-boundary の行・件数経路と MCP で同じ origin 判定と元の UTF-16 座標を維持してください。永続スキーマの変更はありません。 `CSharpOriginContext` はキャンセルに対応してファイル先頭部分ごとに一度だけ構築し、origin の区間を行・一致間で共有します。同じ字句走査で schema の引数位置を追跡し、同時に開いている builder 呼び出しは最大 64、遡及範囲は 64 行とし、超過時は対象ラベルを不明にします。regex/help 分類は開始行ごとにキャッシュし、ラベル照会時にもキャンセルを確認してください。一致やリテラルごとに先行行を再走査したり schema コンテキストを再構築したりしないでください。 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 7a9a7a8d8..58c30f93d 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -106,6 +106,8 @@ EOF; Windows runs it through the synchronous-pipe adapter. ## C# multiline search-origin coverage +`OriginContinuationIssue5348Tests` covers three lexical windows, comments and ordinary/verbatim/raw interpolations, malformed/missing context, Unicode and zero-width coordinates, row/count/search/find parity, cursor pass binding, character limits, cancellation and generation invalidation. The existing #5307 chunk-limit fixture also resumes with two passes. Run these with #5307/#5321/#5324 and CLI help/completion tests on net8/net9; retain one-pass defaults and unknown/partial controls. + Keep cross-chunk closing evidence independent of row limits in database, CLI counts and MCP tests; include schema alignment commas/nested builders and ordinary-format newline rejection with verbatim/raw controls. Issue #5321 extends this fixture with interpolation recovery before row/count/recipe/MCP matches and chunk-budget boundaries. The focused classifier matrix covers expression code, nested strings/comments, ordinary/verbatim/raw forms, escaped braces, malformed delimiters, format text, nesting limits, Unicode coordinates and bounded unavailability metadata. Run both net8/net9 lanes with the #5307 negative controls. @@ -1489,6 +1491,8 @@ net8/net9 で #3831 の末尾保持と JSON 成功・失敗時の収集完了フ ## C# 複数行検索 origin の検証 +`OriginContinuationIssue5348Tests` は 3 つの字句処理窓、コメント、通常／verbatim／raw 補間、不正・欠落文脈、Unicode とゼロ幅の座標、行／件数／search／find の一致、パス数とカーソルの紐づけ、文字数上限、キャンセル、世代変更による無効化を検証します。既存の #5307 チャンク上限 fixture でも 2 パスで継続します。#5307／#5321／#5324 と CLI のヘルプ・補完テストとともに net8／net9 で実行し、既定の 1 パス動作と unknown／partial の対照を維持してください。 + チャンクをまたぐ終端の証拠が行数上限に依存しないことを、DB・CLI 件数・MCP のテストで維持してください。schema の配置指定カンマ／入れ子の builder と、通常書式での改行拒否および verbatim/raw の対照も含めます。 Issue #5321 は同じ fixture に、行／件数／recipe／MCP の一致より前の補間回復とチャンク上限境界を追加します。分類器の専用マトリクスでは、式のコード、入れ子の文字列／コメント、通常／verbatim／raw 形式、波括弧のエスケープ、不正な区切り、書式部分、入れ子の上限、Unicode 座標、上限付きの分類不能メタデータを検証します。#5307 の負例とともに net8/net9 の両方で実行してください。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 8f2e06780..b56599954 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -2342,7 +2342,7 @@ Search origin filters and facets carry C# block-comment, verbatim-string and raw Interpolation nesting and expression delimiters each have a depth limit of 64. Unbalanced or unsupported interpolation (including quoted/braced format components) remains `unknown` from the outer string onward. Unknown C# match facets expose `origin_unavailable` (MCP: `originUnavailable`) with a fixed `reason`, one-based `start_line` / `start_column` (MCP: `startLine` / `startColumn`) and `extent=remaining_file` (`line` for mismatched indexed text). Inspect these facets without an origin filter, or with `--origin unknown`; a code-only zero count cannot establish absence when classification is unavailable. -Classification reads an indexed prefix bounded by 4,096 lines, 8,388,608 UTF-16 characters and 128 chunks per file (overlapping chunk characters count toward the read budget). Missing prefix lines or exhausted bounds produce `unknown`, which does not satisfy `--origin code`. It does not read live source or require a rebuild. Inspect unknown matches without an origin filter when reviewing incomplete or large files. +Classification defaults to one indexed-prefix pass bounded by 4,096 lines, 8,388,608 UTF-16 characters and 128 chunks per file (overlapping chunk characters count toward the read budget). CLI `search`, `audit`, and `find --regex` accept `--origin-passes <1-16>` to explicitly continue lexical state through additional windows with the same per-pass limits. Restart without `--cursor` when changing the pass count. `origin_unavailable` adds optional `retry_origin_passes` and `recovery_guidance`; find counts/terminals also report `origin_passes` and `classification_incomplete_reasons`. Missing/conflicting lines, malformed constructs and exhausted bounds stay `unknown`, which does not satisfy `--origin code`. State is confined to one indexed snapshot and discarded on generation changes; live source reads, schema migration and rebuild are unnecessary. MCP retains one pass. See [bounded continuation and remaining limits](docs/find-scan-controls.md#bounded-c-lexical-continuation-5348). #### Shell search origins @@ -6420,7 +6420,7 @@ function CreateUser src/Services/UserService.cs: 補間の入れ子と式の区切りの深さには、それぞれ 64 の上限があります。不均衡または未対応の補間（引用符や波括弧を含む書式部分など）は、外側の文字列以降を `unknown` とします。不明な C# の一致 facet には `origin_unavailable`（MCP: `originUnavailable`）を付け、固定の `reason`、1 始まりの `start_line` / `start_column`（MCP: `startLine` / `startColumn`）、`extent=remaining_file`（インデックス済みテキストの不一致では `line`）を示します。origin フィルターを外すか `--origin unknown` で確認してください。分類できない場合、コードのみの件数がゼロでも不存在を証明できません。 -分類は各ファイルのインデックス済み先頭部分を、4,096 行、UTF-16 で 8,388,608 文字、128 チャンクを上限として読み取ります（重複チャンクの文字も読み取り上限に含みます）。先頭からの行が欠けている場合や上限を超える場合は `unknown` となり、`--origin code` には一致しません。実ファイルの読み取りや rebuild は不要です。不完全なファイルや大きなファイルのレビューでは、origin フィルターを外して不明な一致も確認してください。 +分類は既定で各ファイルの索引済み先頭部分を 1 パス読み取り、4,096 行、UTF-16 で 8,388,608 文字、128 チャンクを上限とします（重複チャンクの文字も読み取り上限に含みます）。CLI の `search`、`audit`、`find --regex` では `--origin-passes <1-16>` を明示し、各パスの上限を維持したまま次の窓へ字句状態を引き継げます。パス数を変える場合は `--cursor` を外して再開始してください。`origin_unavailable` に任意の `retry_origin_passes` と `recovery_guidance` が加わり、find の件数・終端は `origin_passes` と `classification_incomplete_reasons` も示します。行の欠落・不整合、不正な構文、上限超過は `unknown` のままで、`--origin code` には一致しません。状態は同じ索引スナップショット内に限定し、世代変更時に破棄します。実ソースの読み取り、スキーマ移行、rebuild は不要です。MCP は 1 パスを維持します。[上限付き継続と残る制限](docs/find-scan-controls.md#上限付き-c-字句分類の継続-5348)も参照してください。 #### Shell検索の由来分類 diff --git a/changelog.d/unreleased/5348.fixed.md b/changelog.d/unreleased/5348.fixed.md new file mode 100644 index 000000000..4e7b9bbc4 --- /dev/null +++ b/changelog.d/unreleased/5348.fixed.md @@ -0,0 +1,17 @@ +--- +category: fixed +issues: + - 5348 +affected: + - src/CodeIndex/Database/SearchMatchClassifier.CSharp.cs + - src/CodeIndex/Database/DbSearchReader.CSharpOrigins.cs + - src/CodeIndex/Cli/QueryCommandRunner.Find.cs +--- + +## English + +- **Continue bounded C# origin classification beyond the initial prefix (#5348)** — `search`, `audit`, and `find --regex` accept `--origin-passes` (1–16, default 1). Each additional pass resumes lexical state within the indexed snapshot while retaining the 4,096-line, 8 Mi-character and 128-chunk limits. Unknown diagnostics identify exhausted budgets and bounded retry actions; missing or malformed context remains unknown, and find origin filters, counts and cursor pages retain conservative completeness semantics. + +## 日本語 + +- **C# の origin 分類を最初のプレフィックス以降へ上限付きで継続可能に (#5348)** — `search`、`audit`、`find --regex` に `--origin-passes`（1〜16、既定 1）を追加しました。追加パスは同じ索引スナップショット内で字句状態を引き継ぎ、各パスの 4,096 行・8 Mi 文字・128 チャンクの上限を維持します。unknown の診断は到達した上限と上限付き再試行を案内し、文脈の欠落や不正な構文は unknown のまま扱い、find の origin フィルター・件数・カーソルページで保守的な完全性判定を維持します。 diff --git a/docs/find-scan-controls.md b/docs/find-scan-controls.md index a14197a47..ecde3ef46 100644 --- a/docs/find-scan-controls.md +++ b/docs/find-scan-controls.md @@ -2,6 +2,35 @@ ## English +### Bounded C# lexical continuation (#5348) + +For `search`, `audit`, and `find --regex`, explicitly request `--origin-passes ` +(1–16, default 1) to continue C# lexical state across additional indexed windows. +For example, rerun `cdidx find return --regex --path large.cs --origin code +--origin-passes 2 --count --json`. Each pass still reads at most 4,096 lines, +8 Mi UTF-16 characters (including chunk overlap), and 128 chunks per file. +Up to 16 passes retain at most 65,536 lines / 128 Mi source characters per file; +result limits and page offsets do not change this budget. Smaller character/chunk +windows can advance too, provided at least one complete new line is available. + +Lexical state is resumed within that invocation's indexed snapshot; no checkpoint +survives the query and no schema migration or reindex is required. Every new query +replays its requested passes from the beginning. The pass setting binds cursors and +recipe replay: restart **without `--cursor`** when changing it. Comments, string +delimiters, interpolation frames and schema context carry across windows; results +are classified only after the requested bounded work finishes. Indexed generation +changes discard the provisional context. Live source edits require normal indexing. + +Unknown facets expose `origin_unavailable.reason` and, when another pass may help, +`retry_origin_passes` plus `recovery_guidance`. Find terminal/count output also +exposes `origin_passes`, `classification_incomplete_reasons`, and optional +`retry_origin_passes`. A retry may make further bounded progress without completing +the file. Missing/conflicting chunks, malformed constructs, an individual line or +chunk prefix that cannot fit, and the 16-pass ceiling can still prevent completion; +inspect those regions manually. Find retains unknowns and partial exit `11`, +including explicit `--origin unknown` and unknown matches rejected by filters. +MCP search retains its existing one-pass behavior and shared classifier. + ### Regex origin filters (#5324) Use `cdidx find 'XmlReader\.Create' --regex --path src/ --origin code --json`. @@ -81,6 +110,31 @@ text or JSON output when context from `--before`, `--after`, or ## 日本語 +### 上限付き C# 字句分類の継続 (#5348) + +`search`、`audit`、`find --regex` では `--origin-passes `（1〜16、既定 1）を +明示すると、索引済みの次の窓へ C# の字句状態を引き継げます。例えば +`cdidx find return --regex --path large.cs --origin code --origin-passes 2 --count --json` +で再実行します。各パスは引き続きファイルごとに最大 4,096 行、チャンクの重複分を含む +8 Mi UTF-16 文字、128 チャンクです。最大 16 パスで保持するソースはファイルごとに +65,536 行／128 Mi 文字以内で、結果件数やページ位置で上限は変わりません。文字数・チャンク数で +窓が小さくなっても、完全な新しい行を 1 行以上取得できれば前進できます。 + +状態の継続は同じ呼び出しの索引スナップショット内に限定し、クエリを越えてチェックポイントを +保持しません。スキーマ移行や再索引は不要で、新しいクエリは指定パス数を先頭から再実行します。 +パス数はカーソルと recipe 再実行の条件に含まれるため、変更するときは **`--cursor` を外して** +再開始してください。コメント、文字列の区切り、補間フレーム、schema の文脈を窓の間で引き継ぎ、 +指定した上限付き処理が終了してから結果を分類します。索引世代が変われば暫定文脈を破棄します。 +実ソースの編集を反映するには通常の索引更新が必要です。 + +unknown の facet は `origin_unavailable.reason` を返し、追加パスが役立つ場合は +`retry_origin_passes` と `recovery_guidance` も返します。find の終端・件数出力には +`origin_passes`、`classification_incomplete_reasons`、任意の `retry_origin_passes` が加わります。 +再試行は前進してもファイル全体を完了できるとは限りません。チャンクの欠落・不整合、不正な構文、 +窓に収まらない単独行やチャンク先頭、16 パスの上限で完了できない部分は手動で確認してください。 +`--origin unknown` の明示指定やフィルターで除外した unknown を含め、unknown と partial 終了コード +`11` は find で維持します。MCP search は共通分類器を使い、従来の 1 パス動作を維持します。 + ### 正規表現の origin フィルター (#5324) `cdidx find 'XmlReader\.Create' --regex --path src/ --origin code --json` を使います。 diff --git a/src/CodeIndex/Cli/CliFlagSchema.cs b/src/CodeIndex/Cli/CliFlagSchema.cs index 2fcce4b4c..d314e8c49 100644 --- a/src/CodeIndex/Cli/CliFlagSchema.cs +++ b/src/CodeIndex/Cli/CliFlagSchema.cs @@ -715,6 +715,7 @@ private static IReadOnlyList BuildAll() new() { Name = "--exclude-comments", Description = "Search/Audit/Find --regex: suppress comment-only matches after origin classification", PrimaryCommands = Set("search", "audit", "find") }, new() { Name = "--exclude-strings", Description = "Search/Audit/Find --regex: suppress string, regex, and help-text matches after origin classification", PrimaryCommands = Set("search", "audit", "find") }, new() { Name = "--exclude-fixtures", Description = "Search/Audit/Find --regex: suppress fixture-only matches in tests after origin classification", PrimaryCommands = Set("search", "audit", "find") }, + new() { Name = "--origin-passes", ValuePlaceholder = "", Description = "Search/Audit/Find --regex: C# lexical passes per file (1-16; default 1), each bounded at 4096 lines, 8 Mi characters and 128 chunks; restart without cursor to change", PrimaryCommands = Set("search", "audit", "find") }, new() { Name = "--progress", Description = "Audit --all/Unused: opt in to bounded stderr progress, including captured JSON output; quiet/no-progress take precedence", PrimaryCommands = Set("audit", "unused") }, new() { Name = "--analysis-timeout-ms", ValuePlaceholder = "", Description = "Unused: analysis time budget in milliseconds (1-600000; default 30000); independent of output limits", PrimaryCommands = Set("unused") }, new() { Name = "--continuation", ValuePlaceholder = "", Description = "Audit --all: resume bounded recipe/query observations with an unchanged index, scope, recipes, and --limit", PrimaryCommands = Set("audit") }, diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index 4a25b0cbc..09e181b64 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -83,7 +83,7 @@ private static readonly (string Command, string Usage)[] CommandUsageLines = ("index-commits", "cdidx index --commits [commit-ref ...] [--db ] [--verbose] [--dry-run [--dry-run-path-limit ]] [--json] [--allow-partial] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]"), ("index-changed-between", "cdidx index --changed-between [--db ] [--verbose] [--dry-run [--dry-run-path-limit ]] [--json] [--allow-partial] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]"), ("index-files", "cdidx index --files [path ...] [--db ] [--verbose] [--dry-run [--dry-run-path-limit ]] [--json] [--allow-partial] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]"), - ("search", "cdidx search |--query |-- |--named-query = [--named-query = ...]|--recipe [--include-query ] [--exclude-query ]|--list-recipes [--query ] [--names|--summary-only] [--cursor ] [--audit-scope ] [--source-only] [--show-excluded] [--db ] [--json[=ndjson|array]] [--pretty] [--format ] [--open-issues ] [--repo ] [--duplicate-confidence |--duplicate-threshold ] [--issue-title ] [--issue-label <label>] [--verbose] [--limit <n>|--top <n>|--max-results <n>] [--total-limit <n>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--exclude-comments] [--exclude-strings] [--exclude-fixtures] [--snippet-lines <n>] [--snippet-focus <leftmost|quality|proximity>] [--max-line-width <n>] [--fts] [--exact|--exact-substring|--token-boundary] [--prefix] [--count] [--group-by <file|symbol|origin|return-type|subsystem>] [--since <datetime>] [--no-dedup] [--no-visibility-rank] [--require-before <query>] [--require-after <query>] [--reject-before <query>] [--reject-after <query>] [--guard-window <n>] [--guard-scope <window|same-line|same-symbol>] [--unique <path|file|symbol|origin|return-type|subsystem>] [--count-by <path|file|symbol|origin|return-type|subsystem>] [--origin <origin>] [--match-origin <origin>] [--exclude-origin <origin>] [--result-kind <kind>] [--search-fields <csv>] [--results-only] [--first-per-file] [--sample <n>] [--per-file-limit <n>] [--max-json-bytes <n>] [--allow-partial] [--next-steps]"), + ("search", "cdidx search <query>|--query <query>|-- <query>|--named-query <name>=<query> [--named-query <name>=<query> ...]|--recipe <name|name/query> [--include-query <name>] [--exclude-query <name>]|--list-recipes [--query <filter>] [--names|--summary-only] [--cursor <cursor>] [--audit-scope <source|production-and-tooling|all>] [--source-only] [--show-excluded] [--db <path>] [--json[=ndjson|array]] [--pretty] [--format <text|json|count|compact|grouped|csv|tsv|lsp|qf|sarif|issue-drafts>] [--open-issues <path|github|github:owner/name>] [--repo <owner/name>] [--duplicate-confidence <low|medium|high>|--duplicate-threshold <score>] [--issue-title <title>] [--issue-label <label>] [--verbose] [--limit <n>|--top <n>|--max-results <n>] [--total-limit <n>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--exclude-comments] [--exclude-strings] [--exclude-fixtures] [--origin-passes <n>] [--snippet-lines <n>] [--snippet-focus <leftmost|quality|proximity>] [--max-line-width <n>] [--fts] [--exact|--exact-substring|--token-boundary] [--prefix] [--count] [--group-by <file|symbol|origin|return-type|subsystem>] [--since <datetime>] [--no-dedup] [--no-visibility-rank] [--require-before <query>] [--require-after <query>] [--reject-before <query>] [--reject-after <query>] [--guard-window <n>] [--guard-scope <window|same-line|same-symbol>] [--unique <path|file|symbol|origin|return-type|subsystem>] [--count-by <path|file|symbol|origin|return-type|subsystem>] [--origin <origin>] [--match-origin <origin>] [--exclude-origin <origin>] [--result-kind <kind>] [--search-fields <csv>] [--results-only] [--first-per-file] [--sample <n>] [--per-file-limit <n>] [--max-json-bytes <n>] [--allow-partial] [--next-steps]"), ("recipes", "cdidx recipes [list] [--query <filter>] [--names|--summary-only] [--json] [--pretty] [--format <text|json|compact>] [--max-json-bytes <n>]"), ("recipes-list", "cdidx recipes list [--query <filter>] [--names|--summary-only] [--json] [--pretty] [--format <text|json|compact>] [--max-json-bytes <n>]"), ("audit-baseline-export", "cdidx audit baseline-export <baseline.json> [--recipe <name>] [--db <path>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--audit-scope <source|production-and-tooling|all>] [--since <datetime>] [--limit <n>] [--total-limit <n>] [--overwrite] [--json]"), diff --git a/src/CodeIndex/Cli/QueryCommandRunner.ArgumentParser.Locations.cs b/src/CodeIndex/Cli/QueryCommandRunner.ArgumentParser.Locations.cs index bf94c47be..b53a15cf1 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.ArgumentParser.Locations.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.ArgumentParser.Locations.cs @@ -57,6 +57,17 @@ private bool TryParseLocationOption(string normalizedArg, string currentArg, str case "--exclude-fixtures": excludeFixtures = true; break; + case "--origin-passes": + if (!TryReadRawOptionValue(args, ref i, normalizedArg, inlineValue, out var passesValue, out var passesError)) + AddParseError(passesError!); + else if (!int.TryParse(passesValue, out var passes) || passes < 1 || passes > SearchMatchClassifier.CSharpOriginPassLimit) + AddParseError("Error: --origin-passes must be between 1 and 16."); + else + { + WarnIfDuplicateSingleValueOption(normalizedArg, passesValue!); + originPasses = passes; + } + break; case "--actionable": unusedActionable = true; break; diff --git a/src/CodeIndex/Cli/QueryCommandRunner.ArgumentParser.cs b/src/CodeIndex/Cli/QueryCommandRunner.ArgumentParser.cs index c19c82290..b1d2250d8 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.ArgumentParser.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.ArgumentParser.cs @@ -89,6 +89,7 @@ private sealed partial class QueryArgumentParser private bool excludeComments; private bool excludeStrings; private bool excludeFixtures; + private int originPasses = 1; private List<string>? parseErrors; private bool searchGuardValidationError; private bool missingNumericOptionValue; @@ -528,6 +529,7 @@ private QueryCommandOptions BuildOptions( ExcludeComments = excludeComments, ExcludeStrings = excludeStrings, ExcludeFixtures = excludeFixtures, + OriginPasses = originPasses, ExactName = exactName, ExactSubstring = exactSubstring, TokenBoundary = tokenBoundary, diff --git a/src/CodeIndex/Cli/QueryCommandRunner.AuditAll.cs b/src/CodeIndex/Cli/QueryCommandRunner.AuditAll.cs index 425b7438a..fd8f8399a 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.AuditAll.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.AuditAll.cs @@ -1233,6 +1233,8 @@ void AddValue(string name, string? value) args.Add("--exclude-strings"); if (options.ExcludeFixtures) args.Add("--exclude-fixtures"); + if (options.OriginPasses != 1) + AddReplayValueOption(args, "--origin-passes", options.OriginPasses.ToString(CultureInfo.InvariantCulture)); foreach (var origin in options.MatchOrigins) AddValue("--origin", origin); foreach (var origin in options.ExcludeOrigins) diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs b/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs index bf30bea36..63f6109ac 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs @@ -518,7 +518,8 @@ private static BatchOutputKind ClassifyBatchOutput(string commandName, string[] || JsonEnvelopeWrapper.HasArgument(commandName, args, "--result-kind") || JsonEnvelopeWrapper.HasArgument(commandName, args, "--exclude-comments") || JsonEnvelopeWrapper.HasArgument(commandName, args, "--exclude-strings") - || JsonEnvelopeWrapper.HasArgument(commandName, args, "--exclude-fixtures"))) + || JsonEnvelopeWrapper.HasArgument(commandName, args, "--exclude-fixtures") + || JsonEnvelopeWrapper.HasArgument(commandName, args, "--origin-passes"))) return BatchOutputKind.Ndjson; return BatchOutputKind.JsonDocument; diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Database.cs b/src/CodeIndex/Cli/QueryCommandRunner.Database.cs index ebc6b7427..f35ac79b9 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Database.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Database.cs @@ -90,6 +90,7 @@ private static int WithDb( } reader.IncludeGenerated = options.IncludeGenerated; + reader.OriginPasses = options.OriginPasses; options.InvocationGenerationFingerprint = options.Json && options.JsonOutputFormat == JsonOutputFormatNdjson && !options.ResultsOnly diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Find.cs b/src/CodeIndex/Cli/QueryCommandRunner.Find.cs index 592b4675d..41ab12c41 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Find.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Find.cs @@ -10,7 +10,7 @@ namespace CodeIndex.Cli; public static partial class QueryCommandRunner { internal const int MaxFindLineScanLimit = 10_000_000; - private const string FindUsage = "Usage: cdidx find <query> (--path <glob>|--all) [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <next_cursor>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--exclude-path <glob>] [--exclude-tests] [--context <n>] [--before <n>] [--after <n>] [--snippet-lines <n>] [--focus-line <line>] [--focus-column <n>] [--max-line-width <n>] [--line-scan-limit <n>] [--allow-partial] [--exact] [--regex] [--origin <origin>] [--exclude-origin <origin>] [--result-kind <kind>] [--exclude-comments] [--exclude-strings] [--exclude-fixtures] [--count]\n cdidx find --query <query> (--path <glob>|--all) [...]\n cdidx find [options] -- <query>"; + private const string FindUsage = "Usage: cdidx find <query> (--path <glob>|--all) [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <next_cursor>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--exclude-path <glob>] [--exclude-tests] [--context <n>] [--before <n>] [--after <n>] [--snippet-lines <n>] [--focus-line <line>] [--focus-column <n>] [--max-line-width <n>] [--line-scan-limit <n>] [--allow-partial] [--exact] [--regex] [--origin <origin>] [--exclude-origin <origin>] [--result-kind <kind>] [--exclude-comments] [--exclude-strings] [--exclude-fixtures] [--origin-passes <n>] [--count]\n cdidx find --query <query> (--path <glob>|--all) [...]\n cdidx find [options] -- <query>"; public static int RunFind( string[] cmdArgs, @@ -151,7 +151,7 @@ public static int RunFind( return CommandExitCodes.UsageError; } - var semanticFilters = HasSearchOriginFilters(options) + var semanticFilters = HasFindOriginClassification(options) ? new FindSemanticFilters(options.MatchOrigins, options.ExcludeOrigins, options.ResultKinds, options.ExcludeComments, options.ExcludeStrings, options.ExcludeFixtures) : null; @@ -760,13 +760,19 @@ private static void AddFindScanJsonFields(JsonObject payload, FindScanSummary sc { payload["origin_classification_complete"] = scan.UnknownOriginMatches == 0; payload["unknown_origin_matches"] = scan.UnknownOriginMatches; + payload["origin_passes"] = scan.OriginPasses; if (scan.UnknownOriginMatches > 0) { payload["authoritative_count"] = false; payload["authoritative_rows"] = false; payload["partial_result"] = true; payload["classification_incomplete_reason"] = "origin_classification_unavailable"; - payload["classification_recovery_guidance"] = "Inspect unknown matches without semantic exclusions; classification supports bounded C# and line-local shell context. Absence is not authoritative."; + payload["classification_incomplete_reasons"] = new JsonArray((scan.OriginIncompleteReasons ?? []).Select(reason => JsonValue.Create(reason)).ToArray()); + if (scan.RetryOriginPasses is { } passes) + payload["retry_origin_passes"] = passes; + payload["classification_recovery_guidance"] = scan.RetryOriginPasses is { } retry + ? $"Restart without --cursor using --origin-passes {retry} for another bounded C# lexical pass. Missing/malformed context remains unknown; inspect those matches without exclusions." + : "Inspect unknown matches without semantic exclusions; missing/malformed context or the maximum lexical budget cannot be resolved by more passes. Absence is not authoritative."; } } payload["candidate_files"] = scan.CandidateFiles; @@ -786,8 +792,12 @@ private static void AddFindScanJsonFields(JsonObject payload, FindScanSummary sc payload["line_scan_limit"] = scan.LineLimit.Value; } + private static bool HasFindOriginClassification(QueryCommandOptions options) + => HasSearchOriginFilters(options) || options.OriginPasses != 1 + || JsonEnvelopeWrapper.HasArgument("find", options.InvocationArgs, "--origin-passes"); + private static bool IsFindAllNdjson(QueryCommandOptions options) - => (options.All || HasSearchOriginFilters(options)) + => (options.All || HasFindOriginClassification(options)) && options.Json && options.OutputFormat == OutputFormatJson && options.JsonOutputFormat == JsonOutputFormatNdjson; @@ -913,11 +923,16 @@ private static void WriteFindScanSummary( ? $"; authoritative_count={(!scan.Truncated && !resumedCountPage && scan.UnknownOriginMatches == 0).ToString().ToLowerInvariant()}" : $"; authoritative_rows={(scanComplete && scan.UnknownOriginMatches == 0).ToString().ToLowerInvariant()}"; if (scan.UnknownOriginMatches > 0) + { summary += $"; unknown_origin_matches={scan.UnknownOriginMatches}; absence is not authoritative"; + summary += $"; classification_incomplete_reasons={string.Join(",", scan.OriginIncompleteReasons ?? [])}"; + } var continuationAction = FindScanContinuationAction(scan, resultLimitReached); if (continuationAction != null) summary += $"; continuation_action={continuationAction}"; CommandErrorWriter.WriteStderr($"({summary})"); + if (scan.RetryOriginPasses is { } passes) + CommandErrorWriter.WriteStderr($"Hint: restart without --cursor using --origin-passes {passes} for another bounded C# lexical pass; missing/malformed context remains unknown."); if (nextCursor != null) CommandErrorWriter.WriteStderr($"next_cursor={nextCursor}"); var recoveryGuidance = FindScanRecoveryGuidance(scan, resultLimitReached); diff --git a/src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs b/src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs index 4c928a0e9..61ca12103 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs @@ -254,6 +254,8 @@ private static JsonObject BuildQueryContextJson(QueryCommandOptions options, Jso query["exclude_strings"] = true; if (options.ExcludeFixtures) query["exclude_fixtures"] = true; + if (options.OriginPasses != 1) + query["origin_passes"] = options.OriginPasses; var generatedFileFilterAvailable = ActiveSqliteDiagnosticsReader.Value?.GeneratedFileFilterAvailable; query["include_generated"] = options.IncludeGenerated; query["generated_code_policy"] = options.IncludeGenerated diff --git a/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs b/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs index 05c363fd9..0432a8b02 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs @@ -1774,6 +1774,8 @@ private static void AddSearchRecipeCompactReplayOptions( args.Add("--exclude-strings"); if (options.ExcludeFixtures) args.Add("--exclude-fixtures"); + if (options.OriginPasses != 1) + AddReplayValueOption(args, "--origin-passes", options.OriginPasses.ToString(CultureInfo.InvariantCulture)); foreach (var origin in options.MatchOrigins) AddReplayValueOption(args, "--origin", origin); foreach (var origin in options.ExcludeOrigins) @@ -5777,6 +5779,8 @@ private static string BuildSearchRecipeReplayCommand( args.Add("--exclude-strings"); if (options.ExcludeFixtures) args.Add("--exclude-fixtures"); + if (options.OriginPasses != 1) + AddReplayValueOption(args, "--origin-passes", options.OriginPasses.ToString(CultureInfo.InvariantCulture)); foreach (var origin in options.MatchOrigins) AddReplayValueOption(args, "--origin", origin); foreach (var origin in options.ExcludeOrigins) @@ -6159,6 +6163,8 @@ private static string BuildAdHocSearchReplayCommand(QueryCommandOptions options, args.Add("--exclude-strings"); if (options.ExcludeFixtures) args.Add("--exclude-fixtures"); + if (options.OriginPasses != 1) + AddReplayValueOption(args, "--origin-passes", options.OriginPasses.ToString(CultureInfo.InvariantCulture)); if (options.Since.HasValue) AddReplayValueOption(args, "--since", options.Since.Value.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture)); if (options.NoDedup) diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index a237ea4ca..d9aee1fbc 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -166,6 +166,7 @@ public sealed class QueryCommandOptions public bool ExcludeComments { get; init; } public bool ExcludeStrings { get; init; } public bool ExcludeFixtures { get; init; } + public int OriginPasses { get; init; } = 1; public bool ExactName { get; init; } public bool ExactSubstring { get; init; } public bool TokenBoundary { get; init; } diff --git a/src/CodeIndex/Database/DbSearchReader.CSharpOrigins.cs b/src/CodeIndex/Database/DbSearchReader.CSharpOrigins.cs new file mode 100644 index 000000000..cd4f6030c --- /dev/null +++ b/src/CodeIndex/Database/DbSearchReader.CSharpOrigins.cs @@ -0,0 +1,100 @@ +namespace CodeIndex.Database; + +public partial class DbReader +{ + private int _originPasses = 1; + internal Action<int>? OriginWindowStartingForTesting { get; set; } + internal int OriginPasses + { + get => _originPasses; + set + { + ArgumentOutOfRangeException.ThrowIfLessThan(value, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(value, SearchMatchClassifier.CSharpOriginPassLimit); + _originPasses = value; + } + } + + private (long Local, long External) ReadOriginGeneration() + { + ThrowIfCancellationRequested(); + using var command = _conn.CreateCommand(); + command.CommandText = "SELECT total_changes(), data_version FROM pragma_data_version"; + using var reader = command.ExecuteTrackedReader(); + reader.TrackedRead(); + return (reader.GetInt64(0), reader.GetInt64(1)); + } + + private SearchMatchClassifier.CSharpOriginWindow ReadCSharpOriginWindow( + string path, int firstLine, (long Local, long External) generation, CancellationToken cancellation) + { + cancellation.ThrowIfCancellationRequested(); + OriginWindowStartingForTesting?.Invoke(firstLine); + cancellation.ThrowIfCancellationRequested(); + var lines = new Dictionary<int, string>(); + if (ReadOriginGeneration() != generation) + return new(lines, "indexed_generation_changed"); + var endLine = firstLine + SearchMatchClassifier.CSharpContextLineLimit - 1; + var budget = SearchMatchClassifier.CSharpContextCharacterLimit; + var chunks = 0; + var conflictLine = int.MaxValue; + var reason = "indexed_prefix_unavailable"; + using var command = _conn.CreateCommand(); + command.CommandText = @" + SELECT c.start_line, c.end_line, substr(c.content, 1, @characters + 1) + FROM chunks c JOIN files f ON c.file_id = f.id + WHERE f.path = @path AND c.start_line <= @endLine AND c.end_line >= @firstLine + ORDER BY c.start_line, c.id LIMIT @chunks"; + SqliteCommandPolicy.Add(command, "@path", path); + SqliteCommandPolicy.Add(command, "@firstLine", firstLine); + SqliteCommandPolicy.Add(command, "@endLine", endLine); + SqliteCommandPolicy.Add(command, "@characters", budget); + SqliteCommandPolicy.Add(command, "@chunks", SearchMatchClassifier.CSharpContextChunkLimit); + using var reader = command.ExecuteTrackedReader(); + while (reader.TrackedRead()) + { + cancellation.ThrowIfCancellationRequested(); + ThrowIfCancellationRequested(); + chunks++; + if (reader.IsDBNull(2)) + break; + var start = reader.GetInt32(0); + var last = Math.Min(endLine, reader.GetInt32(1)); + if (start < 1 || last < start) + break; + var content = reader.GetString(2); + var truncated = content.Length > budget; + if (truncated) + { + // Charge overlap too; a partial physical line never becomes evidence. + var newline = budget == 0 ? -1 : content.LastIndexOf('\n', budget - 1, budget); + content = newline < 0 ? string.Empty : content[..(newline + 1)]; + reason = "character_budget_exhausted"; + } + budget -= content.Length; + var lastOffset = truncated ? Math.Min(last - start, content.Count(ch => ch == '\n') - 1) : last - start; + foreach (var (offset, value) in EnumerateContentLines(content, Math.Max(0, firstLine - start), lastOffset)) + { + cancellation.ThrowIfCancellationRequested(); + if (!lines.TryAdd(start + offset, value) && lines[start + offset] != value) + conflictLine = Math.Min(conflictLine, start + offset); + } + if (truncated) + break; + } + if (conflictLine != int.MaxValue) + { + lines.Remove(conflictLine); + reason = "indexed_text_mismatch"; + } + else if (chunks == SearchMatchClassifier.CSharpContextChunkLimit && reason != "character_budget_exhausted") + reason = "chunk_budget_exhausted"; + // A hole inside materialized evidence cannot be repaired by skipping to a later window. + var contiguousEnd = firstLine; + while (lines.ContainsKey(contiguousEnd)) + contiguousEnd++; + if (lines.Keys.Any(line => line > contiguousEnd) && conflictLine == int.MaxValue) + reason = "indexed_prefix_unavailable"; + return new(lines, reason); + } +} diff --git a/src/CodeIndex/Database/DbSearchReader.cs b/src/CodeIndex/Database/DbSearchReader.cs index 07c007c19..d50ce5c15 100644 --- a/src/CodeIndex/Database/DbSearchReader.cs +++ b/src/CodeIndex/Database/DbSearchReader.cs @@ -389,47 +389,17 @@ FROM fts_chunks private void AttachCSharpOriginLines(List<SearchResult> results, CancellationToken cancellationToken = default) { - // Read only bounded indexed prefixes; never fill holes with invented blank lines. - // Share each prefix between its rows. Per-file budgets must not depend on pagination. + // Every row/page uses the same explicit pass budget and indexed generation. foreach (var group in results.Where(r => string.Equals(r.Lang, "csharp", StringComparison.OrdinalIgnoreCase)) .GroupBy(r => r.Path, StringComparer.Ordinal)) { - // Closing interpolation evidence can occur after the last returned match. - var endLine = SearchMatchClassifier.CSharpContextLineLimit; - var budget = SearchMatchClassifier.CSharpContextCharacterLimit; - var lines = new Dictionary<int, string>(); - using var cmd = _conn.CreateCommand(); - cmd.CommandText = @" - SELECT c.start_line, substr(c.content, 1, @characters + 1) - FROM chunks c JOIN files f ON c.file_id = f.id - WHERE f.path = @path AND c.start_line <= @endLine - ORDER BY c.start_line, c.id LIMIT @chunks"; - SqliteCommandPolicy.Add(cmd, "@path", group.Key); - SqliteCommandPolicy.Add(cmd, "@endLine", endLine); - SqliteCommandPolicy.Add(cmd, "@characters", budget); - SqliteCommandPolicy.Add(cmd, "@chunks", SearchMatchClassifier.CSharpContextChunkLimit); - using var reader = cmd.ExecuteTrackedReader(); - while (reader.TrackedRead()) - { - ThrowIfCancellationRequested(); - cancellationToken.ThrowIfCancellationRequested(); - var content = reader.GetString(1); - var truncated = content.Length > budget; - if (truncated) - { - // A capped row can still prove complete preceding lines. Never admit its partial tail. - var lastNewline = budget == 0 ? -1 : content.LastIndexOf('\n', budget - 1, budget); - content = lastNewline < 0 ? string.Empty : content[..(lastNewline + 1)]; - } - budget -= content.Length; - var start = reader.GetInt32(0); - var lastOffset = truncated ? Math.Min(endLine - start, content.Count(ch => ch == '\n') - 1) : endLine - start; - foreach (var (offset, value) in EnumerateContentLines(content, 0, lastOffset)) - lines.TryAdd(start + offset, value); - if (truncated) - break; - } - var origins = new SearchMatchClassifier.CSharpOriginContext(group.Key, lines, cancellationToken.CanBeCanceled ? cancellationToken : _cancellation); + var generation = ReadOriginGeneration(); + var token = cancellationToken.CanBeCanceled ? cancellationToken : _cancellation; + var origins = new SearchMatchClassifier.CSharpOriginContext(group.Key, + start => ReadCSharpOriginWindow(group.Key, start, generation, token), OriginPasses, token); + if (ReadOriginGeneration() != generation) + origins = new SearchMatchClassifier.CSharpOriginContext(group.Key, + _ => new(new Dictionary<int, string>(), "indexed_generation_changed"), 1, token); foreach (var result in group) result.CSharpOrigins = origins; } diff --git a/src/CodeIndex/Database/IndexedFindModels.cs b/src/CodeIndex/Database/IndexedFindModels.cs index 56ddac50f..516bcac63 100644 --- a/src/CodeIndex/Database/IndexedFindModels.cs +++ b/src/CodeIndex/Database/IndexedFindModels.cs @@ -91,6 +91,9 @@ private sealed class FindScanState(FindResumePosition resume) { internal int UnknownOriginMatches { get; set; } internal bool ClassificationApplied { get; set; } + internal int OriginPasses { get; set; } = 1; + internal SortedSet<string> OriginIncompleteReasons { get; } = new(StringComparer.Ordinal); + internal int? RetryOriginPasses { get; set; } internal int CandidateFiles { get; set; } internal int FilesScanned { get; set; } internal int LinesScanned { get; set; } @@ -173,6 +176,9 @@ internal FindScanSummary CreateSummary( NextByteOffset, ResultLimitReached, ClassificationApplied, - UnknownOriginMatches); + UnknownOriginMatches, + OriginIncompleteReasons.ToArray(), + RetryOriginPasses, + OriginPasses); } } diff --git a/src/CodeIndex/Database/IndexedFindPipeline.cs b/src/CodeIndex/Database/IndexedFindPipeline.cs index 71676309b..d64291e44 100644 --- a/src/CodeIndex/Database/IndexedFindPipeline.cs +++ b/src/CodeIndex/Database/IndexedFindPipeline.cs @@ -58,6 +58,7 @@ private FindSearchPlan ScanFiles( FindScanState state) { state.ClassificationApplied = request.SemanticFilters is not null; + state.OriginPasses = _owner.OriginPasses; var comparison = request.Exact ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; var regexMatcher = request.Regex ? CreateFindRegexMatcher(request.Query, request.Exact) @@ -167,7 +168,12 @@ private bool ScanFileLines( { var facet = ClassifyFindMatch(file, indexedLine, lineMatch, origins); if (facet.Origin == SearchMatchClassifier.Unknown) + { state.UnknownOriginMatches++; + state.OriginIncompleteReasons.Add(facet.OriginUnavailable?.Reason ?? "unsupported_origin_context"); + if (facet.OriginUnavailable?.RetryOriginPasses is { } retry) + state.RetryOriginPasses = Math.Max(state.RetryOriginPasses ?? 0, retry); + } if (!filters.Accepts(facet)) { matchOrdinal++; diff --git a/src/CodeIndex/Database/SearchMatchClassifier.CSharp.cs b/src/CodeIndex/Database/SearchMatchClassifier.CSharp.cs index b51caa14e..d4250139c 100644 --- a/src/CodeIndex/Database/SearchMatchClassifier.CSharp.cs +++ b/src/CodeIndex/Database/SearchMatchClassifier.CSharp.cs @@ -6,6 +6,10 @@ internal static partial class SearchMatchClassifier // Leave room above the existing 4 Mi-character semantic-analysis windows. internal const int CSharpContextCharacterLimit = 8 * 1024 * 1024; internal const int CSharpContextChunkLimit = 128; + internal const int CSharpOriginPassLimit = 16; + + internal sealed record CSharpOriginWindow(IReadOnlyDictionary<int, string> Lines, + string StopReason = "indexed_prefix_unavailable"); private static string ClassifyCSharpContext( string path, int line, string text, int index, IReadOnlyDictionary<int, string>? context) @@ -22,13 +26,23 @@ internal sealed class CSharpOriginContext private int _unknownLine = int.MaxValue; private int _unknownColumn; private string _unknownReason = "indexed_prefix_unavailable"; + private int? _retryPasses; public CSharpOriginContext(string path, IReadOnlyDictionary<int, string> context, CancellationToken cancellation = default) + : this(path, _ => new CSharpOriginWindow(context), 1, cancellation) + { + } + + // Continuations are private to this construction and its indexed snapshot. Never + // publish provisional interpolation origins or reuse lexical state across queries. + public CSharpOriginContext(string path, Func<int, CSharpOriginWindow> readWindow, + int passLimit, CancellationToken cancellation = default) { + ArgumentOutOfRangeException.ThrowIfLessThan(passLimit, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(passLimit, CSharpOriginPassLimit); _cancellation = cancellation; var schema = IsSchemaDescriptionPath(path) ? new CSharpSchemaCalls() : null; - var remaining = CSharpContextCharacterLimit; var state = 0; // code, block comment, verbatim string, raw string, ordinary string, format var quotes = 0; var dollars = 0; @@ -37,240 +51,256 @@ public CSharpOriginContext(string path, IReadOnlyDictionary<int, string> context var pendingLine = 0; var pendingColumn = 0; CSharpStringLabel? label = null; - for (var line = 1; line <= CSharpContextLineLimit; line++) + var line = 1; + for (var pass = 1; pass <= passLimit; pass++) { cancellation.ThrowIfCancellationRequested(); - if (!context.TryGetValue(line, out var source) || source.Length > remaining) - { - SetUnknown(pendingLine > 0 ? pendingLine : line, pendingLine > 0 ? pendingColumn : 0, - source is null ? "indexed_prefix_unavailable" : "character_budget_exhausted"); - return; - } - remaining -= source.Length; - var parsed = new CSharpOriginLine(path, source, schema is not null, cancellation); - _lines.Add(line, parsed); - var i = 0; - while (i < source.Length) + var window = readWindow(line); + var context = window.Lines; + var remaining = CSharpContextCharacterLimit; + var firstLine = line; + var endLine = (long)line + CSharpContextLineLimit; + var stopReason = window.StopReason; + for (; line < endLine; line++) { cancellation.ThrowIfCancellationRequested(); - if ((state == 0 || state == 5) && expressions.TryPeek(out var expression)) + if (!context.TryGetValue(line, out var source) || source.Length > remaining) + { + if (source is not null) + stopReason = "character_budget_exhausted"; + break; + } + remaining -= source.Length; + var parsed = new CSharpOriginLine(path, source, schema is not null, cancellation); + _lines.Add(line, parsed); + var i = 0; + while (i < source.Length) { - var ch = source[i]; - if (ch == '}' && expression.Delimiters.Count == 0) + cancellation.ThrowIfCancellationRequested(); + if ((state == 0 || state == 5) && expressions.TryPeek(out var expression)) { - var run = CountRun(source, i, '}'); - var required = expression.State == 3 ? expression.Dollars : 1; - if (run < required) + var ch = source[i]; + if (ch == '}' && expression.Delimiters.Count == 0) { - SetUnknown(pendingLine, pendingColumn, "unbalanced_interpolation"); - return; + var run = CountRun(source, i, '}'); + var required = expression.State == 3 ? expression.Dollars : 1; + if (run < required) + { + SetUnknown(pendingLine, pendingColumn, "unbalanced_interpolation"); + return; + } + parsed.Add(i, i + required, StringLiteral, expression.Label); + i += required; + state = expression.State; + quotes = expression.Quotes; + dollars = expression.Dollars; + quote = '"'; + label = expression.Label; + schema = expression.Schema; + expressions.Pop(); + continue; } - parsed.Add(i, i + required, StringLiteral, expression.Label); - i += required; - state = expression.State; - quotes = expression.Quotes; - dollars = expression.Dollars; - quote = '"'; - label = expression.Label; - schema = expression.Schema; - expressions.Pop(); - continue; - } - if (state == 5) - { - if (ch is '{' or '"') + if (state == 5) { - SetUnknown(pendingLine, pendingColumn, "unsupported_interpolation_format"); - return; + if (ch is '{' or '"') + { + SetUnknown(pendingLine, pendingColumn, "unsupported_interpolation_format"); + return; + } + parsed.Add(i, i + 1, StringLiteral, expression.Label); + i++; + continue; } - parsed.Add(i, i + 1, StringLiteral, expression.Label); - i++; - continue; - } - if (ch is '(' or '[' or '{') - { - if (expression.Delimiters.Count == 64) + if (ch is '(' or '[' or '{') { - SetUnknown(pendingLine, pendingColumn, "interpolation_nesting_limit"); - return; + if (expression.Delimiters.Count == 64) + { + SetUnknown(pendingLine, pendingColumn, "interpolation_nesting_limit"); + return; + } + expression.Delimiters.Push(ch); } - expression.Delimiters.Push(ch); - } - else if (ch is ')' or ']' or '}') - { - var expected = ch == ')' ? '(' : ch == ']' ? '[' : '{'; - if (!expression.Delimiters.TryPop(out var opener) || opener != expected) + else if (ch is ')' or ']' or '}') { - SetUnknown(pendingLine, pendingColumn, "unbalanced_interpolation"); - return; + var expected = ch == ')' ? '(' : ch == ']' ? '[' : '{'; + if (!expression.Delimiters.TryPop(out var opener) || opener != expected) + { + SetUnknown(pendingLine, pendingColumn, "unbalanced_interpolation"); + return; + } + } + else if (ch == ':' && expression.Delimiters.Count == 0 && + !(i + 1 < source.Length && source[i + 1] == ':') && !(i > 0 && source[i - 1] == ':')) + { + state = 5; + continue; } } - else if (ch == ':' && expression.Delimiters.Count == 0 && - !(i + 1 < source.Length && source[i + 1] == ':') && !(i > 0 && source[i - 1] == ':')) - { - state = 5; - continue; - } - } - if (state == 1 || state == 0 && source.AsSpan(i).StartsWith("/*", StringComparison.Ordinal)) - { - var start = i; - var end = source.IndexOf("*/", state == 1 ? i : i + 2, StringComparison.Ordinal); - state = end < 0 ? 1 : 0; - i = end < 0 ? source.Length : end + 2; - parsed.Add(start, i, Comment); - continue; - } - if (state == 0 && source.AsSpan(i).StartsWith("//", StringComparison.Ordinal)) - { - parsed.Add(i, source.Length, Comment); - break; - } - - var startIndex = i; - var ordinary = state == 4; - if (state == 0) - { - if (source[i] is not ('"' or '\'')) + if (state == 1 || state == 0 && source.AsSpan(i).StartsWith("/*", StringComparison.Ordinal)) { - schema?.Consume(source, i, line); - i++; + var start = i; + var end = source.IndexOf("*/", state == 1 ? i : i + 2, StringComparison.Ordinal); + state = end < 0 ? 1 : 0; + i = end < 0 ? source.Length : end + 2; + parsed.Add(start, i, Comment); continue; } - quote = source[i]; - var run = quote == '"' ? CountRun(source, i, '"') : 1; - var verbatim = quote == '"' && (i > 0 && source[i - 1] == '@' || - i > 1 && source[i - 1] == '$' && source[i - 2] == '@'); - var raw = !verbatim && run >= 3; - dollars = 0; - var prefix = i - 1; - if (prefix >= 0 && source[prefix] == '@') - prefix--; - if (quote == '"') - while (prefix >= 0 && source[prefix--] == '$') - dollars++; - if (dollars > 64) + if (state == 0 && source.AsSpan(i).StartsWith("//", StringComparison.Ordinal)) { - SetUnknown(line, i, "interpolation_nesting_limit"); - return; - } - i += raw ? run : 1; - var schemaOrigin = parsed.HasDescriptionProperty - ? startIndex == parsed.DescriptionQuote ? SchemaDescription : null - : schema?.Exhausted == true ? Unknown - : schema?.IsDescriptionArgument(line) == true ? SchemaDescription : null; - label = new CSharpStringLabel(parsed, schemaOrigin); - state = raw ? 3 : verbatim ? 2 : 4; - ordinary = state == 4; - quotes = run; - if (dollars > 0 && pendingLine == 0) - { - pendingLine = line; - pendingColumn = startIndex; + parsed.Add(i, source.Length, Comment); + break; } - } - var enteredExpression = false; - while (i < source.Length) - { - if ((i & 4095) == 0) - cancellation.ThrowIfCancellationRequested(); - var ch = source[i]; - if (dollars > 0 && ch == '}') + var startIndex = i; + var ordinary = state == 4; + if (state == 0) { - var run = CountRun(source, i, '}'); - if (state == 3 ? run >= dollars : run % 2 != 0) + if (source[i] is not ('"' or '\'')) { - SetUnknown(pendingLine, pendingColumn, "unbalanced_interpolation"); - return; + schema?.Consume(source, i, line); + i++; + continue; } - i += run; - continue; - } - if (dollars > 0 && ch == '{') - { - if (state != 3 && i + 1 < source.Length && source[i + 1] == '{') + quote = source[i]; + var run = quote == '"' ? CountRun(source, i, '"') : 1; + var verbatim = quote == '"' && (i > 0 && source[i - 1] == '@' || + i > 1 && source[i - 1] == '$' && source[i - 2] == '@'); + var raw = !verbatim && run >= 3; + dollars = 0; + var prefix = i - 1; + if (prefix >= 0 && source[prefix] == '@') + prefix--; + if (quote == '"') + while (prefix >= 0 && source[prefix--] == '$') + dollars++; + if (dollars > 64) { - i += 2; - continue; + SetUnknown(line, i, "interpolation_nesting_limit"); + return; } - var braces = 1; - while (state == 3 && braces < dollars && i + braces < source.Length && source[i + braces] == '{') - braces++; - if (state != 3 || braces >= dollars) + i += raw ? run : 1; + var schemaOrigin = parsed.HasDescriptionProperty + ? startIndex == parsed.DescriptionQuote ? SchemaDescription : null + : schema?.Exhausted == true ? Unknown + : schema?.IsDescriptionArgument(line) == true ? SchemaDescription : null; + label = new CSharpStringLabel(parsed, schemaOrigin); + state = raw ? 3 : verbatim ? 2 : 4; + ordinary = state == 4; + quotes = run; + if (dollars > 0 && pendingLine == 0) { - var run = CountRun(source, i, '{'); - if (expressions.Count == 64 || state == 3 && run >= 2 * dollars) - { - SetUnknown(pendingLine, pendingColumn, "interpolation_nesting_limit"); - return; - } - i += state == 3 ? run : 1; - parsed.Add(startIndex, i, StringLiteral, label); - expressions.Push(new InterpolationFrame(state, quotes, dollars, label, schema)); - schema = schema is null ? null : new CSharpSchemaCalls(); - state = 0; - enteredExpression = true; - break; + pendingLine = line; + pendingColumn = startIndex; } - i += braces; - continue; } - if (state == 3 && ch == '"') + + var enteredExpression = false; + while (i < source.Length) { - var run = CountRun(source, i, '"'); - if (run >= quotes) + if ((i & 4095) == 0) + cancellation.ThrowIfCancellationRequested(); + var ch = source[i]; + if (dollars > 0 && ch == '}') { - i += quotes; - state = 0; - break; + var run = CountRun(source, i, '}'); + if (state == 3 ? run >= dollars : run % 2 != 0) + { + SetUnknown(pendingLine, pendingColumn, "unbalanced_interpolation"); + return; + } + i += run; + continue; } - i += run; - continue; - } - if (state == 2 && ch == '"') - { - if (i + 1 < source.Length && source[i + 1] == '"') + if (dollars > 0 && ch == '{') { - i += 2; + if (state != 3 && i + 1 < source.Length && source[i + 1] == '{') + { + i += 2; + continue; + } + var braces = 1; + while (state == 3 && braces < dollars && i + braces < source.Length && source[i + braces] == '{') + braces++; + if (state != 3 || braces >= dollars) + { + var run = CountRun(source, i, '{'); + if (expressions.Count == 64 || state == 3 && run >= 2 * dollars) + { + SetUnknown(pendingLine, pendingColumn, "interpolation_nesting_limit"); + return; + } + i += state == 3 ? run : 1; + parsed.Add(startIndex, i, StringLiteral, label); + expressions.Push(new InterpolationFrame(state, quotes, dollars, label, schema)); + schema = schema is null ? null : new CSharpSchemaCalls(); + state = 0; + enteredExpression = true; + break; + } + i += braces; continue; } - i++; - state = 0; - break; - } - if (ordinary) - { - if (ch == '\\') + if (state == 3 && ch == '"') { - i = Math.Min(source.Length, i + 2); + var run = CountRun(source, i, '"'); + if (run >= quotes) + { + i += quotes; + state = 0; + break; + } + i += run; continue; } - if (ch == quote) + if (state == 2 && ch == '"') { + if (i + 1 < source.Length && source[i + 1] == '"') + { + i += 2; + continue; + } i++; state = 0; break; } + if (ordinary) + { + if (ch == '\\') + { + i = Math.Min(source.Length, i + 2); + continue; + } + if (ch == quote) + { + i++; + state = 0; + break; + } + } + i++; } - i++; + if (!enteredExpression) + parsed.Add(startIndex, i, StringLiteral, label); + if (state == 0 && expressions.Count == 0) + pendingLine = 0; + } + if (state == 4 || state == 5 && expressions.TryPeek(out var format) && format.State == 4) + { + SetUnknown(pendingLine > 0 ? pendingLine : line, pendingLine > 0 ? pendingColumn : 0, + "unterminated_ordinary_string"); + return; } - if (!enteredExpression) - parsed.Add(startIndex, i, StringLiteral, label); - if (state == 0 && expressions.Count == 0) - pendingLine = 0; - } - if (state == 4 || state == 5 && expressions.TryPeek(out var format) && format.State == 4) - { - SetUnknown(pendingLine > 0 ? pendingLine : line, pendingLine > 0 ? pendingColumn : 0, - "unterminated_ordinary_string"); - return; } + if (line == endLine) + stopReason = "line_budget_exhausted"; + var resumable = stopReason is "line_budget_exhausted" or "character_budget_exhausted" or "chunk_budget_exhausted"; + if (resumable && line > firstLine && pass < passLimit) + continue; + if (resumable && line > firstLine && passLimit < CSharpOriginPassLimit) + _retryPasses = passLimit + 1; + SetUnknown(pendingLine > 0 ? pendingLine : line, pendingLine > 0 ? pendingColumn : 0, stopReason); + return; } - if (pendingLine > 0) - SetUnknown(pendingLine, pendingColumn, "unterminated_interpolation"); - else - SetUnknown(CSharpContextLineLimit + 1, 0, "line_budget_exhausted"); } private sealed class InterpolationFrame(int state, int quotes, int dollars, CSharpStringLabel? label, CSharpSchemaCalls? schema) @@ -319,6 +349,10 @@ public SearchOriginUnavailable GetUnavailable(int line, string text) StartLine = _unknownLine == int.MaxValue ? line : _unknownLine, StartColumn = _unknownColumn + 1, Extent = "remaining_file", + RetryOriginPasses = _retryPasses, + RecoveryGuidance = _retryPasses is { } passes + ? $"Restart the query without --cursor using --origin-passes {passes}; each pass retains the 4096-line, 8 Mi-character and 128-chunk limits." + : "Inspect the indexed context for missing or malformed source; additional passes cannot prove this region within the supported bounds.", }; } diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index db5431c0d..b446c6881 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -110,7 +110,10 @@ public readonly record struct FindScanSummary( int? NextByteOffset = null, bool ResultLimitReached = false, bool ClassificationApplied = false, - int UnknownOriginMatches = 0); + int UnknownOriginMatches = 0, + IReadOnlyList<string>? OriginIncompleteReasons = null, + int? RetryOriginPasses = null, + int OriginPasses = 1); public readonly record struct FindCountResult(int Count, int FileCount, FindScanSummary Scan); @@ -247,6 +250,10 @@ public sealed class SearchOriginUnavailable public int StartLine { get; set; } public int StartColumn { get; set; } public string Extent { get; set; } = string.Empty; + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? RetryOriginPasses { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? RecoveryGuidance { get; set; } } public sealed record FtsQueryDiagnostics( diff --git a/tests/CodeIndex.Tests/OriginContinuationIssue5348Tests.cs b/tests/CodeIndex.Tests/OriginContinuationIssue5348Tests.cs new file mode 100644 index 000000000..c11a5e020 --- /dev/null +++ b/tests/CodeIndex.Tests/OriginContinuationIssue5348Tests.cs @@ -0,0 +1,239 @@ +using System.Text.Json; +using CodeIndex.Cli; +using CodeIndex.Database; +using CodeIndex.Models; +using Microsoft.Data.Sqlite; +using static CodeIndex.Tests.QueryCommandTestSupport; + +namespace CodeIndex.Tests; + +[Collection("SQLite pool sensitive")] +public sealed class OriginContinuationIssue5348Tests +{ + [Fact] + public void LexicalWindows_PreserveMultilineStateCoordinatesAndHardLimits() + { + (string Open, string Middle, string Close, string Origin)[] cases = + [ + ("/*", "😀 日本語 Needle", "*/", "comment"), + ("var s = @\"", "😀 \"\"Needle\"\"", "\";", "string_literal"), + ("var s = \"\"\"\"", "😀 \"\"\" Needle", "\"\"\"\";", "string_literal"), + ("var s = $\"{", "日本語.Needle()", "}\";", "code"), + ("var s = $@\"{", "Call($\"{Needle()}\", \"}\")", "}\";", "code"), + ("var s = $$\"\"\"{{", "/* } */ 日本語.Needle()", "}}\"\"\";", "code"), + ("var s = $@\"{value:", "Needle", "}\";", "string_literal"), + ("var s = $\"{value:", "Needle", "}\";", "unknown"), + ("var s = $@\"{", "Needle(]", "}\";", "unknown"), + ("var s = $@\"{", "Needle()", "", "unknown"), + ]; + foreach (var (open, middle, close, expected) in cases) + { + var lines = Enumerable.Repeat("", 8194).ToArray(); + lines[4094] = open; + lines[4096] = middle; + lines[4097] = close; + lines[8192] = "Needle();"; + var starts = new List<int>(); + SearchMatchClassifier.CSharpOriginWindow Read(int start) + { + starts.Add(start); + return new(Enumerable.Range(start, Math.Min(4096, lines.Length - start + 1)) + .ToDictionary(line => line, line => lines[line - 1])); + } + var column = middle.IndexOf("Needle", StringComparison.Ordinal); + var prefix = new SearchMatchClassifier.CSharpOriginContext("src/a.cs", Read, 1); + Assert.Equal("unknown", prefix.GetOrigin(4097, middle, column)); + starts.Clear(); + var resumed = new SearchMatchClassifier.CSharpOriginContext("src/a.cs", Read, 3); + var facet = SearchMatchClassifier.Classify("src/a.cs", "csharp", 4097, middle, + column + 1, 6, csharpContext: resumed); + Assert.True(expected == facet.Origin, $"{open} / {middle}: {facet.Origin}"); + Assert.Equal((4097, column + 1, 6), (facet.Line, facet.Column, facet.Length)); + if (expected != "unknown") + { + Assert.Equal(new[] { 1, 4097, 8193 }, starts); + Assert.Equal("code", resumed.GetOrigin(8193, lines[8192], 0)); + } + else + Assert.Null(facet.OriginUnavailable!.RetryOriginPasses); + } + + var huge = new string(' ', SearchMatchClassifier.CSharpContextCharacterLimit); + var reads = 0; + SearchMatchClassifier.CSharpOriginWindow Characters(int start) + { + reads++; + return new(start == 1 + ? new Dictionary<int, string> { [1] = huge, [2] = "Needle();" } + : new Dictionary<int, string> { [2] = "Needle();" }); + } + var bounded = new SearchMatchClassifier.CSharpOriginContext("src/a.cs", Characters, 1); + Assert.Equal("character_budget_exhausted", bounded.GetUnavailable(2, "Needle();").Reason); + Assert.Equal(2, bounded.GetUnavailable(2, "Needle();").RetryOriginPasses); + reads = 0; + var complete = new SearchMatchClassifier.CSharpOriginContext("src/a.cs", Characters, 2); + Assert.Equal(2, reads); + Assert.Equal("code", complete.GetOrigin(2, "Needle();", 0)); + var stalled = new SearchMatchClassifier.CSharpOriginContext("src/a.cs", + _ => new(new Dictionary<int, string>(), "character_budget_exhausted"), 16); + Assert.Null(stalled.GetUnavailable(1, "Needle").RetryOriginPasses); + Assert.Throws<ArgumentOutOfRangeException>(() => new SearchMatchClassifier.CSharpOriginContext("a.cs", Characters, 17)); + } + + [Fact] + public void Queries_KeepRowsCountsOriginsAndCursorPagesConsistent() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_origin_5348"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + var lines = Enumerable.Repeat("", 8199).ToArray(); + lines[0] = "Needle();"; + lines[4094] = "/*"; + lines[4096] = "😀 Needle"; + lines[4097] = "*/"; + lines[4199] = "日本語.Needle();"; + lines[8190] = "var s = $$\"\"\"{{"; + lines[8192] = "Needle()"; + lines[8193] = "}}\"\"\";"; + Seed(dbPath, lines); + using var db = new DbContext(DbOpenIntent.QueryOnly, dbPath); + using var reader = new DbReader(db); + foreach (var passes in new[] { 1, 2, 3 }) + { + reader.OriginPasses = passes; + var search = SearchSnippetFormatter.ToCompactResults( + reader.Search("Needle", 30, exact: true, deduplicate: false), "Needle", exposeLiteralHighlights: true) + .SelectMany(row => row.MatchFacets).OrderBy(facet => facet.Line).ToArray(); + var find = reader.FindInFiles("Needle", 30, regex: true, semanticFilters: new([], [], [])); + Assert.Equal(search.Select(f => (f.Line, f.Column, f.Length, f.Origin)), + find.Select(row => Assert.Single(row.MatchFacets!)).Select(f => (f.Line, f.Column, f.Length, f.Origin))); + foreach (var origin in new[] { "code", "comment", "unknown" }) + { + var filters = new FindSemanticFilters([origin], [], []); + var expected = search.Where(f => f.Origin == origin).ToArray(); + Assert.Equal(expected.Length, reader.CountFindInFiles("Needle", regex: true, semanticFilters: filters).Count); + Assert.Equal(expected.Select(f => f.Line), reader.FindInFiles("Needle", 30, regex: true, semanticFilters: filters).Select(r => r.Line)); + foreach (var command in new[] { "search", "find" }) + { + var args = new[] { command, "Needle", "--path", "src/a.cs", "--db", dbPath, "--origin", origin, + "--origin-passes", passes.ToString(), "--json", "--count", command == "find" ? "--regex" : "--exact" }; + var (_, output, error) = CaptureConsole(() => ProgramRunner.Run(args, JsonOptions, "test")); + Assert.Empty(error); + using var count = JsonDocument.Parse(output); + Assert.Equal(expected.Length, count.RootElement.GetProperty("count").GetInt32()); + if (command == "find") + { + Assert.Equal(passes == 3, count.RootElement.GetProperty("origin_classification_complete").GetBoolean()); + if (passes < 3) + Assert.Equal(passes + 1, count.RootElement.GetProperty("retry_origin_passes").GetInt32()); + } + } + } + } + + foreach (var (passes, origin, expectedCount) in new[] { (3, "code", 3), (1, "unknown", 3) }) + { + var args = new[] { "find", "(?=Needle)", "--regex", "--path", "src/a.cs", "--db", dbPath, + "--origin", origin, "--origin-passes", passes.ToString(), "--json", "--limit", "1", "--max-json-bytes", "8000", + "--fields", "path,line,column,length,match_facets" }; + var positions = new HashSet<(int, int)>(); + string? cursor = null; + string? firstCursor = null; + for (var page = 0; page < 4; page++) + { + var (exit, output, error) = CaptureConsole(() => ProgramRunner.Run( + cursor is null ? args : [.. args, "--cursor", cursor], JsonOptions, "test")); + Assert.Equal(origin == "unknown" ? CommandExitCodes.PartialResult : 0, exit); + Assert.Empty(error); + using var result = JsonDocument.Parse(output); + var row = Assert.Single(result.RootElement.GetProperty("results").EnumerateArray()); + Assert.Equal(0, row.GetProperty("length").GetInt32()); + Assert.True(positions.Add((row.GetProperty("line").GetInt32(), row.GetProperty("column").GetInt32()))); + cursor = result.RootElement.GetProperty("metadata").GetProperty("next_cursor").GetString(); + firstCursor ??= cursor; + if (cursor is null) + break; + } + Assert.Null(cursor); + Assert.Equal(expectedCount, positions.Count); + var (changed, _, _) = CaptureConsole(() => ProgramRunner.Run( + [.. args, "--origin-passes", "2", "--cursor", firstCursor!], JsonOptions, "test")); + Assert.Equal(CommandExitCodes.UsageError, changed); + } + foreach (var command in new[] { "search", "audit", "find" }) + { + Assert.Contains(CliFlagSchema.GetCompletionFlagsForCommand(command), flag => flag.Name == "--origin-passes"); + foreach (var value in new[] { "0", "17", "-1", "bad", "999999999999999" }) + { + var (exit, output, error) = CaptureConsole(() => ProgramRunner.Run( + [command, "Needle", "--db", dbPath, "--path", "src/a.cs", "--origin-passes=" + value, "--json"], JsonOptions, "test")); + Assert.Equal(CommandExitCodes.UsageError, exit); + Assert.Empty(error); + using var failure = JsonDocument.Parse(output); + } + } + } + + [Fact] + public void Queries_DiscardChangedGenerationAndPreserveMissingMalformedAndCancelledUnknowns() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_origin_state_5348"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + var lines = Enumerable.Repeat("", 4098).ToArray(); + lines[0] = "Needle();"; + lines[4096] = "Needle();"; + Seed(dbPath, lines); + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + using var reader = new DbReader(db.Connection) { OriginPasses = 2 }; + using var cancel = new CancellationTokenSource(); + reader.OriginWindowStartingForTesting = start => { if (start > 1) cancel.Cancel(); }; + Assert.Throws<OperationCanceledException>(() => reader.FindInFiles("Needle", 10, regex: true, + cancellationToken: cancel.Token, semanticFilters: new(["code"], [], []))); + reader.OriginWindowStartingForTesting = null; + Assert.Equal(2, reader.CountFindInFiles("Needle", regex: true, semanticFilters: new(["code"], [], [])).Count); + + void Execute(string sql) + { + using var command = db.Connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } + reader.OriginWindowStartingForTesting = start => + { + if (start > 1) + Execute("UPDATE chunks SET content = '/*' || content WHERE start_line = 1"); + }; + var changed = reader.CountFindInFiles("Needle", regex: true, semanticFilters: new(["code"], [], [])); + Assert.Equal(0, changed.Count); + Assert.Contains("indexed_generation_changed", changed.Scan.OriginIncompleteReasons!); + Assert.Null(changed.Scan.RetryOriginPasses); + reader.OriginWindowStartingForTesting = null; + Assert.Equal(2, reader.CountFindInFiles("Needle", regex: true, semanticFilters: new(["comment"], [], [])).Count); + + Execute("DELETE FROM chunks WHERE start_line = 4001"); + var missing = reader.CountFindInFiles("Needle", regex: true, semanticFilters: new(["unknown"], [], [])); + Assert.Equal(1, missing.Count); + Assert.Contains("indexed_prefix_unavailable", missing.Scan.OriginIncompleteReasons!); + Assert.Null(missing.Scan.RetryOriginPasses); + Execute("UPDATE chunks SET content = '$\"{Call(}\";' WHERE start_line = 1"); + var malformed = reader.CountFindInFiles("Needle", regex: true, semanticFilters: new(["unknown"], [], [])); + Assert.Equal(1, malformed.Count); + Assert.Contains("unbalanced_interpolation", malformed.Scan.OriginIncompleteReasons!); + Assert.Null(malformed.Scan.RetryOriginPasses); + } + + private static void Seed(string dbPath, string[] lines) + { + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + var writer = new DbWriter(db.Connection); + var id = writer.UpsertFile(new FileRecord + { + Path = "src/a.cs", Lang = "csharp", Lines = lines.Length, Size = string.Join('\n', lines).Length, + Modified = DateTime.UtcNow, Checksum = "fixture", + }); + writer.InsertChunks(Enumerable.Range(0, (lines.Length + 79) / 80).Select(index => new ChunkRecord + { + FileId = id, ChunkIndex = index, StartLine = index * 80 + 1, EndLine = Math.Min(lines.Length, (index + 1) * 80), + Content = string.Join('\n', lines.Skip(index * 80).Take(80)), + }).ToList()); + } +} diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSearchIssue5307Tests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSearchIssue5307Tests.cs index 29d7c255c..d12c2d1ff 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSearchIssue5307Tests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSearchIssue5307Tests.cs @@ -209,6 +209,11 @@ public void SearchMatchClassifier_ChunkBudgetSharesOriginsAcrossRows_Issue5307() Assert.Equal(full.CSharpOrigins!.GetOrigin(page.StartLine, page.Content, column), page.CSharpOrigins!.GetOrigin(page.StartLine, page.Content, column)); } + reader.OriginPasses = 2; + var resumed = reader.Search("ArgumentList", count + 1, exact: true, deduplicate: false); + Assert.Equal(count, resumed.Count); + Assert.All(SearchSnippetFormatter.ToCompactResults(resumed, "ArgumentList", exposeLiteralHighlights: true), + row => Assert.Equal("code", Assert.Single(row.MatchFacets).Origin)); } finally { From a88d7f1737a3eb20a567c8a5ef8b6844a4d3a278 Mon Sep 17 00:00:00 2001 From: Widthdom <widthdom@gmail.com> Date: Sun, 13 Sep 2026 00:07:29 +0900 Subject: [PATCH 2/2] Reject conflicting origin continuation overlap (#5348) --- DEVELOPER_GUIDE.md | 4 ++ TESTING_GUIDE.md | 4 +- changelog.d/unreleased/5348.fixed.md | 4 +- docs/find-scan-controls.md | 15 ++++--- .../Database/DbSearchReader.CSharpOrigins.cs | 22 +++++++--- src/CodeIndex/Database/DbSearchReader.cs | 15 +++++-- .../OriginContinuationIssue5348Tests.cs | 44 +++++++++++++++++++ 7 files changed, 89 insertions(+), 19 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index ebe8c2bef..bd688c86c 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -60,6 +60,8 @@ untruncated output. Attestation still relies on the verifier's exit status. Issue #5348 adds explicit `--origin-passes` (1–16, default 1) to CLI search/audit/regex-find. `DbSearchReader.CSharpOrigins.cs` supplies successive bounded windows to one `CSharpOriginContext`; lexical locals, interpolation frames, labels and schema state survive each window. Limits apply per pass, including overlapping chunk characters, with at most 65,536 lines / 128 Mi source characters retained per file. State never survives a query; local `total_changes()` and external `data_version` checks discard contexts if their indexed generation changes. Preserve missing/conflicting-line holes, cancellation, fixed per-query pass budgets, cursor/replay binding, and bounded retry diagnostics. A zero-progress window cannot request another pass. See [continuation behavior](docs/find-scan-controls.md#bounded-c-lexical-continuation-5348). +Compare newly encountered overlap with retained bounded evidence before resuming. Conflicting overlap discards the entire provisional context: earlier interpolation decisions may depend on its closing text. + Read the bounded prefix independently of the last returned chunk so later closing delimiters cannot change earlier origins across pages. Interpolation frames isolate schema-call state (including alignment commas) while retaining nested builder detection. Ordinary interpolation format text cannot cross a physical newline; verbatim/raw formats can. Issue #5321 resumes ordinary/verbatim/raw interpolation with at most 64 active interpolation frames and 64 balanced delimiters per expression. Nested comments/strings are consumed by the same cancellable prefix pass. Merge adjacent literal spans, preserve UTF-16 coordinates, and keep unsupported formats or unbalanced/missing context unknown with bounded `SearchMatchFacet.OriginUnavailable` reason/start/extent evidence. No work budget depends on requested matches or pagination. @@ -4627,6 +4629,8 @@ JSON の `installer_output_incomplete` に記録します。判明した親の Issue #5348 は CLI の search／audit／regex-find に明示的な `--origin-passes`（1〜16、既定 1）を追加します。`DbSearchReader.CSharpOrigins.cs` は同じ `CSharpOriginContext` へ上限付きの窓を順次渡し、字句状態、補間フレーム、ラベル、schema の文脈を引き継ぎます。重複チャンクの文字数を含む上限はパスごとに適用し、保持するソースはファイルごとに最大 65,536 行／128 Mi 文字です。クエリを越えて状態を保持せず、同一接続の `total_changes()` と外部変更の `data_version` によって索引世代の変化時に文脈を破棄します。欠落・不整合な行、キャンセル、クエリごとに固定したパス数、カーソル・再実行条件の紐づけ、上限付き再試行診断を維持してください。前進できない窓では追加パスを案内しません。[継続動作の説明](docs/find-scan-controls.md#上限付き-c-字句分類の継続-5348)も参照してください。 +継続前に、新しく読み取った重複部分を保持済みの上限付き証拠と照合します。不一致があれば暫定文脈全体を破棄します。先行する補間の判定が、その部分の閉じ区切りに依存する場合があるためです。 + 返す最後のチャンクとは独立して上限付きの先頭部分を読み取り、後続の閉じ区切りによってページ間で先行箇所の origin が変わらないようにします。補間フレームは配置指定のカンマを含む schema 呼び出し状態を分離し、入れ子の builder 検出も維持します。通常の補間書式部分では物理改行を許可せず、verbatim/raw の書式部分では許可します。 Issue #5321 は、同時に開いている補間フレームを最大 64、式ごとの対応する区切りを最大 64 として、通常／verbatim／raw 補間から走査を再開します。入れ子のコメントと文字列も同じキャンセル可能な先頭部分の走査で処理します。隣接するリテラル区間を結合し、UTF-16 座標を保持してください。未対応の書式や不均衡・欠落した文脈は、上限付きの `SearchMatchFacet.OriginUnavailable` の理由・開始位置・範囲を伴う不明状態にします。処理上限は一致数やページングに依存しません。 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 58c30f93d..801073154 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -106,7 +106,7 @@ EOF; Windows runs it through the synchronous-pipe adapter. ## C# multiline search-origin coverage -`OriginContinuationIssue5348Tests` covers three lexical windows, comments and ordinary/verbatim/raw interpolations, malformed/missing context, Unicode and zero-width coordinates, row/count/search/find parity, cursor pass binding, character limits, cancellation and generation invalidation. The existing #5307 chunk-limit fixture also resumes with two passes. Run these with #5307/#5321/#5324 and CLI help/completion tests on net8/net9; retain one-pass defaults and unknown/partial controls. +`OriginContinuationIssue5348Tests` covers three lexical windows, comments and ordinary/verbatim/raw interpolations, malformed/missing context, Unicode and zero-width coordinates, row/count/search/find parity, cursor pass binding, character limits, cancellation, generation invalidation, and conflicting/consistent overlaps at the 128-chunk boundary. The existing #5307 chunk-limit fixture also resumes with two passes. Run these with #5307/#5321/#5324 and CLI help/completion tests on net8/net9; retain one-pass defaults and unknown/partial controls. Keep cross-chunk closing evidence independent of row limits in database, CLI counts and MCP tests; include schema alignment commas/nested builders and ordinary-format newline rejection with verbatim/raw controls. @@ -1491,7 +1491,7 @@ net8/net9 で #3831 の末尾保持と JSON 成功・失敗時の収集完了フ ## C# 複数行検索 origin の検証 -`OriginContinuationIssue5348Tests` は 3 つの字句処理窓、コメント、通常／verbatim／raw 補間、不正・欠落文脈、Unicode とゼロ幅の座標、行／件数／search／find の一致、パス数とカーソルの紐づけ、文字数上限、キャンセル、世代変更による無効化を検証します。既存の #5307 チャンク上限 fixture でも 2 パスで継続します。#5307／#5321／#5324 と CLI のヘルプ・補完テストとともに net8／net9 で実行し、既定の 1 パス動作と unknown／partial の対照を維持してください。 +`OriginContinuationIssue5348Tests` は 3 つの字句処理窓、コメント、通常／verbatim／raw 補間、不正・欠落文脈、Unicode とゼロ幅の座標、行／件数／search／find の一致、パス数とカーソルの紐づけ、文字数上限、キャンセル、世代変更による無効化、128 チャンク境界で重複行が不一致・一致する場合を検証します。既存の #5307 チャンク上限 fixture でも 2 パスで継続します。#5307／#5321／#5324 と CLI のヘルプ・補完テストとともに net8／net9 で実行し、既定の 1 パス動作と unknown／partial の対照を維持してください。 チャンクをまたぐ終端の証拠が行数上限に依存しないことを、DB・CLI 件数・MCP のテストで維持してください。schema の配置指定カンマ／入れ子の builder と、通常書式での改行拒否および verbatim/raw の対照も含めます。 diff --git a/changelog.d/unreleased/5348.fixed.md b/changelog.d/unreleased/5348.fixed.md index 4e7b9bbc4..b6e06dd2a 100644 --- a/changelog.d/unreleased/5348.fixed.md +++ b/changelog.d/unreleased/5348.fixed.md @@ -10,8 +10,8 @@ affected: ## English -- **Continue bounded C# origin classification beyond the initial prefix (#5348)** — `search`, `audit`, and `find --regex` accept `--origin-passes` (1–16, default 1). Each additional pass resumes lexical state within the indexed snapshot while retaining the 4,096-line, 8 Mi-character and 128-chunk limits. Unknown diagnostics identify exhausted budgets and bounded retry actions; missing or malformed context remains unknown, and find origin filters, counts and cursor pages retain conservative completeness semantics. +- **Continue bounded C# origin classification beyond the initial prefix (#5348)** — `search`, `audit`, and `find --regex` accept `--origin-passes` (1–16, default 1). Each additional pass resumes lexical state within the indexed snapshot while retaining the 4,096-line, 8 Mi-character and 128-chunk limits. Generation changes or conflicting overlap discard the provisional context. Unknown diagnostics identify exhausted budgets and bounded retry actions; missing or malformed context remains unknown, and find origin filters, counts and cursor pages retain conservative completeness semantics. ## 日本語 -- **C# の origin 分類を最初のプレフィックス以降へ上限付きで継続可能に (#5348)** — `search`、`audit`、`find --regex` に `--origin-passes`（1〜16、既定 1）を追加しました。追加パスは同じ索引スナップショット内で字句状態を引き継ぎ、各パスの 4,096 行・8 Mi 文字・128 チャンクの上限を維持します。unknown の診断は到達した上限と上限付き再試行を案内し、文脈の欠落や不正な構文は unknown のまま扱い、find の origin フィルター・件数・カーソルページで保守的な完全性判定を維持します。 +- **C# の origin 分類を最初のプレフィックス以降へ上限付きで継続可能に (#5348)** — `search`、`audit`、`find --regex` に `--origin-passes`（1〜16、既定 1）を追加しました。追加パスは同じ索引スナップショット内で字句状態を引き継ぎ、各パスの 4,096 行・8 Mi 文字・128 チャンクの上限を維持します。索引世代の変更や重複行の不一致があれば暫定文脈を破棄します。unknown の診断は到達した上限と上限付き再試行を案内し、文脈の欠落や不正な構文は unknown のまま扱い、find の origin フィルター・件数・カーソルページで保守的な完全性判定を維持します。 diff --git a/docs/find-scan-controls.md b/docs/find-scan-controls.md index ecde3ef46..44f9bc52d 100644 --- a/docs/find-scan-controls.md +++ b/docs/find-scan-controls.md @@ -7,8 +7,9 @@ For `search`, `audit`, and `find --regex`, explicitly request `--origin-passes <n>` (1–16, default 1) to continue C# lexical state across additional indexed windows. For example, rerun `cdidx find return --regex --path large.cs --origin code ---origin-passes 2 --count --json`. Each pass still reads at most 4,096 lines, -8 Mi UTF-16 characters (including chunk overlap), and 128 chunks per file. +--origin-passes 2 --count --json`. Each pass classifies at most 4,096 new lines +and reads at most 8 Mi UTF-16 characters (including chunk overlap) and 128 chunks +per file. Previously retained overlap is compared without replaying lexical state. Up to 16 passes retain at most 65,536 lines / 128 Mi source characters per file; result limits and page offsets do not change this budget. Smaller character/chunk windows can advance too, provided at least one complete new line is available. @@ -19,7 +20,9 @@ replays its requested passes from the beginning. The pass setting binds cursors recipe replay: restart **without `--cursor`** when changing it. Comments, string delimiters, interpolation frames and schema context carry across windows; results are classified only after the requested bounded work finishes. Indexed generation -changes discard the provisional context. Live source edits require normal indexing. +changes discard the provisional context. Conflicting overlap also discards the +entire context, because earlier interpolation decisions may depend on its closing +text. Live source edits require normal indexing. Unknown facets expose `origin_unavailable.reason` and, when another pass may help, `retry_origin_passes` plus `recovery_guidance`. Find terminal/count output also @@ -115,8 +118,9 @@ text or JSON output when context from `--before`, `--after`, or `search`、`audit`、`find --regex` では `--origin-passes <n>`（1〜16、既定 1）を 明示すると、索引済みの次の窓へ C# の字句状態を引き継げます。例えば `cdidx find return --regex --path large.cs --origin code --origin-passes 2 --count --json` -で再実行します。各パスは引き続きファイルごとに最大 4,096 行、チャンクの重複分を含む -8 Mi UTF-16 文字、128 チャンクです。最大 16 パスで保持するソースはファイルごとに +で再実行します。各パスで新しく分類する行はファイルごとに最大 4,096 行、読み取る量は +チャンクの重複分を含む 8 Mi UTF-16 文字、128 チャンクです。保持済みの重複部分は +字句状態を再実行せず照合します。最大 16 パスで保持するソースはファイルごとに 65,536 行／128 Mi 文字以内で、結果件数やページ位置で上限は変わりません。文字数・チャンク数で 窓が小さくなっても、完全な新しい行を 1 行以上取得できれば前進できます。 @@ -125,6 +129,7 @@ text or JSON output when context from `--before`, `--after`, or パス数はカーソルと recipe 再実行の条件に含まれるため、変更するときは **`--cursor` を外して** 再開始してください。コメント、文字列の区切り、補間フレーム、schema の文脈を窓の間で引き継ぎ、 指定した上限付き処理が終了してから結果を分類します。索引世代が変われば暫定文脈を破棄します。 +重複行が不一致の場合も、先行する補間の判定がその閉じ区切りに依存し得るため、文脈全体を破棄します。 実ソースの編集を反映するには通常の索引更新が必要です。 unknown の facet は `origin_unavailable.reason` を返し、追加パスが役立つ場合は diff --git a/src/CodeIndex/Database/DbSearchReader.CSharpOrigins.cs b/src/CodeIndex/Database/DbSearchReader.CSharpOrigins.cs index cd4f6030c..2b1a8a444 100644 --- a/src/CodeIndex/Database/DbSearchReader.CSharpOrigins.cs +++ b/src/CodeIndex/Database/DbSearchReader.CSharpOrigins.cs @@ -26,7 +26,8 @@ internal int OriginPasses } private SearchMatchClassifier.CSharpOriginWindow ReadCSharpOriginWindow( - string path, int firstLine, (long Local, long External) generation, CancellationToken cancellation) + string path, int firstLine, (long Local, long External) generation, + Dictionary<int, string> retainedLines, CancellationToken cancellation) { cancellation.ThrowIfCancellationRequested(); OriginWindowStartingForTesting?.Invoke(firstLine); @@ -73,19 +74,26 @@ private SearchMatchClassifier.CSharpOriginWindow ReadCSharpOriginWindow( } budget -= content.Length; var lastOffset = truncated ? Math.Min(last - start, content.Count(ch => ch == '\n') - 1) : last - start; - foreach (var (offset, value) in EnumerateContentLines(content, Math.Max(0, firstLine - start), lastOffset)) + foreach (var (offset, value) in EnumerateContentLines(content, 0, lastOffset)) { cancellation.ThrowIfCancellationRequested(); - if (!lines.TryAdd(start + offset, value) && lines[start + offset] != value) - conflictLine = Math.Min(conflictLine, start + offset); + var number = start + offset; + if (retainedLines.TryGetValue(number, out var previous) && previous != value) + conflictLine = Math.Min(conflictLine, number); + if (number >= firstLine) + { + lines.TryAdd(number, value); + retainedLines.TryAdd(number, value); + } } if (truncated) break; } if (conflictLine != int.MaxValue) { - lines.Remove(conflictLine); - reason = "indexed_text_mismatch"; + // Earlier origins can depend on closing evidence in the overlap. The caller + // discards the entire provisional context, including completed interpolations. + return new(new Dictionary<int, string>(), "indexed_text_mismatch"); } else if (chunks == SearchMatchClassifier.CSharpContextChunkLimit && reason != "character_budget_exhausted") reason = "chunk_budget_exhausted"; @@ -93,7 +101,7 @@ private SearchMatchClassifier.CSharpOriginWindow ReadCSharpOriginWindow( var contiguousEnd = firstLine; while (lines.ContainsKey(contiguousEnd)) contiguousEnd++; - if (lines.Keys.Any(line => line > contiguousEnd) && conflictLine == int.MaxValue) + if (lines.Keys.Any(line => line > contiguousEnd)) reason = "indexed_prefix_unavailable"; return new(lines, reason); } diff --git a/src/CodeIndex/Database/DbSearchReader.cs b/src/CodeIndex/Database/DbSearchReader.cs index d50ce5c15..85a015fa9 100644 --- a/src/CodeIndex/Database/DbSearchReader.cs +++ b/src/CodeIndex/Database/DbSearchReader.cs @@ -395,11 +395,20 @@ private void AttachCSharpOriginLines(List<SearchResult> results, CancellationTok { var generation = ReadOriginGeneration(); var token = cancellationToken.CanBeCanceled ? cancellationToken : _cancellation; + var retainedLines = new Dictionary<int, string>(); + var conflictingContext = false; var origins = new SearchMatchClassifier.CSharpOriginContext(group.Key, - start => ReadCSharpOriginWindow(group.Key, start, generation, token), OriginPasses, token); - if (ReadOriginGeneration() != generation) + start => + { + var window = ReadCSharpOriginWindow(group.Key, start, generation, retainedLines, token); + conflictingContext |= window.StopReason == "indexed_text_mismatch"; + return window; + }, OriginPasses, token); + var invalidReason = ReadOriginGeneration() != generation ? "indexed_generation_changed" + : conflictingContext ? "indexed_text_mismatch" : null; + if (invalidReason is not null) origins = new SearchMatchClassifier.CSharpOriginContext(group.Key, - _ => new(new Dictionary<int, string>(), "indexed_generation_changed"), 1, token); + _ => new(new Dictionary<int, string>(), invalidReason), 1, token); foreach (var result in group) result.CSharpOrigins = origins; } diff --git a/tests/CodeIndex.Tests/OriginContinuationIssue5348Tests.cs b/tests/CodeIndex.Tests/OriginContinuationIssue5348Tests.cs index c11a5e020..0ae9d19de 100644 --- a/tests/CodeIndex.Tests/OriginContinuationIssue5348Tests.cs +++ b/tests/CodeIndex.Tests/OriginContinuationIssue5348Tests.cs @@ -219,6 +219,50 @@ void Execute(string sql) Assert.Equal(1, malformed.Count); Assert.Contains("unbalanced_interpolation", malformed.Scan.OriginIncompleteReasons!); Assert.Null(malformed.Scan.RetryOriginPasses); + + Execute("DELETE FROM chunks"); + var writer = new DbWriter(db.Connection); + var fileId = writer.UpsertFile(new FileRecord + { + Path = "src/a.cs", Lang = "csharp", Lines = 129, Size = 129, + Modified = DateTime.UtcNow, Checksum = "overlap", + }); + writer.InsertChunks(Enumerable.Range(1, 128).Select(line => new ChunkRecord + { + FileId = fileId, ChunkIndex = line - 1, StartLine = line, EndLine = line, + Content = line == 128 ? "/*" : "", + }).Append(new ChunkRecord + { + FileId = fileId, ChunkIndex = 128, StartLine = 128, EndLine = 129, + Content = "// changed\nNeedle();", + }).ToList()); + var conflicting = reader.CountFindInFiles("Needle", regex: true, semanticFilters: new(["code"], [], [])); + Assert.Equal(0, conflicting.Count); + Assert.Equal(1, conflicting.Scan.UnknownOriginMatches); + Assert.Contains("indexed_text_mismatch", conflicting.Scan.OriginIncompleteReasons!); + Assert.Null(conflicting.Scan.RetryOriginPasses); + Assert.Single(reader.FindInFiles("Needle", 10, regex: true, semanticFilters: new(["unknown"], [], []))); + var searchConflict = Assert.Single(SearchSnippetFormatter.ToCompactResults( + reader.Search("Needle", 10, exact: true), "Needle", exposeLiteralHighlights: true)); + Assert.Equal("unknown", Assert.Single(searchConflict.MatchFacets).Origin); + + Execute("UPDATE chunks SET content = '/*\nNeedle();' WHERE chunk_index = 128"); + var consistent = reader.CountFindInFiles("Needle", regex: true, semanticFilters: new(["comment"], [], [])); + Assert.Equal(1, consistent.Count); + Assert.Equal(0, consistent.Scan.UnknownOriginMatches); + + Execute("UPDATE chunks SET content = '$@\"{' WHERE chunk_index = 125"); + Execute("UPDATE chunks SET content = 'Needle()' WHERE chunk_index = 126"); + Execute("UPDATE chunks SET content = '}\";' WHERE chunk_index = 127"); + Execute("UPDATE chunks SET content = '// changed\nNeedle();' WHERE chunk_index = 128"); + var completedInterpolation = reader.CountFindInFiles("Needle", regex: true, semanticFilters: new(["code"], [], [])); + Assert.Equal(0, completedInterpolation.Count); + Assert.Equal(2, completedInterpolation.Scan.UnknownOriginMatches); + Assert.Contains("indexed_text_mismatch", completedInterpolation.Scan.OriginIncompleteReasons!); + Execute("UPDATE chunks SET content = '}\";\nNeedle();' WHERE chunk_index = 128"); + var restoredInterpolation = reader.CountFindInFiles("Needle", regex: true, semanticFilters: new(["code"], [], [])); + Assert.Equal(2, restoredInterpolation.Count); + Assert.Equal(0, restoredInterpolation.Scan.UnknownOriginMatches); } private static void Seed(string dbPath, string[] lines)