From 5c80bc44ebd9879e792f9d62321a4e7b8859bee3 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 21:54:34 +0000 Subject: [PATCH] Test phase 8: graph checked on monorepo layouts; known gaps listed Generated repositories shaped like real monorepos, checked against grimp and TypeScript: - scripts/graph-oracle/layouts.py: Python projects side by side each with its own src/ (monorepo), a root folder that is a package (root-init), and a namespace package split over two folders (namespace); - scripts/graph-oracle/ts_layouts.mjs: npm workspaces installed as symlinks, tsconfig.base.json paths that projects extend, barrels, .mts/.cts, import type, import(), require, and the forms fcs does not handle (#imports, exports subpaths, a folder's own package.json main). Baselines become a list: benches/graph/known_gaps.toml names every edge fcs misses or draws wrongly on each CI corpus (Django, Vite, the deep trees, the layouts), with its reason (a row of the graph-support page) and exact count. graph_oracle --gaps/--corpus fails on an edge that fits no gap and on a count that changed. The Python oracle builds one graph across ROOT and ROOT/src, plus FCS_ORACLE_PY_PATHS and FCS_ORACLE_PY_NAMESPACES for the layouts. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EkCcM7SF9X6MgaCjoysdAw --- .github/workflows/graph-oracle.yml | 46 ++++- CHANGELOG.md | 12 ++ benches/graph/known_gaps.toml | 85 +++++++++ docs/book/src/how-it-works/testing.md | 2 +- docs/book/src/reference/graph-support.md | 27 ++- examples/graph_oracle.rs | 162 ++++++++++++++++- scripts/graph-oracle/layouts.py | 196 +++++++++++++++++++++ scripts/graph-oracle/ts_layouts.mjs | 215 +++++++++++++++++++++++ src/check/oracle/py.py | 56 ++++-- 9 files changed, 762 insertions(+), 39 deletions(-) create mode 100644 benches/graph/known_gaps.toml create mode 100644 scripts/graph-oracle/layouts.py create mode 100644 scripts/graph-oracle/ts_layouts.mjs diff --git a/.github/workflows/graph-oracle.yml b/.github/workflows/graph-oracle.yml index 4bf687f4..293fc061 100644 --- a/.github/workflows/graph-oracle.yml +++ b/.github/workflows/graph-oracle.yml @@ -1,10 +1,13 @@ name: graph oracle # Does the dependency graph draw the edges the languages' own resolvers -# draw? examples/graph_oracle.rs (the comparison `fcs check` runs) compares fcs's import graph with grimp on -# Django and with TypeScript's resolveModuleName on Vite, and fails when a -# change misses or adds more edges than the recorded baseline. Lower the -# baselines when a fix closes a gap. +# draw? examples/graph_oracle.rs (the comparison `fcs check` runs) compares +# fcs's import graph with grimp on Django, generated deep trees and +# generated monorepo layouts, and with TypeScript's resolveModuleName on +# Vite and a generated workspace monorepo. Every edge fcs misses or draws +# wrongly must be a known gap listed in benches/graph/known_gaps.toml (each +# tied to a row of the book's graph-support page), with its exact count: a +# new miss fails, and so does a fix until its gap is lowered or removed. on: push: @@ -15,6 +18,7 @@ on: - "src/**" - "examples/graph_oracle.rs" - "scripts/graph-oracle/**" + - "benches/graph/**" - "tests/python_codebase.rs" - "tests/support/**" - "Cargo.toml" @@ -77,21 +81,36 @@ jobs: run: | python src/check/oracle/py.py bench-corpus/django > py.json ./target/release/examples/graph_oracle bench-corpus/django --oracle py.json --lang py \ - --max-missing 0 --max-extra 0 --markdown graph-py.md + --gaps benches/graph/known_gaps.toml --corpus django --markdown graph-py.md - name: Python (grimp on a package nested 14 levels deep) run: | python scripts/graph-oracle/deep_python_tree.py deep-py 14 python src/check/oracle/py.py deep-py > py-deep.json ./target/release/examples/graph_oracle deep-py --oracle py-deep.json --lang py \ - --max-missing 0 --max-extra 0 --markdown graph-py-deep.md + --gaps benches/graph/known_gaps.toml --corpus deep-py --markdown graph-py-deep.md - name: Python (grimp on a repository whose root folder has __init__.py) run: | python scripts/graph-oracle/deep_python_tree.py root-init-py 4 --root-init python src/check/oracle/py.py root-init-py > py-root-init.json ./target/release/examples/graph_oracle root-init-py --oracle py-root-init.json --lang py \ - --max-missing 0 --max-extra 0 --markdown graph-py-root-init.md + --gaps benches/graph/known_gaps.toml --corpus root-init-py --markdown graph-py-root-init.md + + # Repositories laid out like real monorepos, generated with seed 1 + # (scripts/graph-oracle/layouts.py): projects side by side each with + # its own src/, a root folder that is a package, and a namespace + # package split over two folders. oracle.env tells grimp what is on + # sys.path. + - name: Python (generated monorepo layouts) + run: | + for layout in monorepo root-init namespace; do + python scripts/graph-oracle/layouts.py "$layout" "py-$layout" --seed 1 + (set -a; . "py-$layout/oracle.env"; set +a + python src/check/oracle/py.py "py-$layout" > "py-$layout.json") + ./target/release/examples/graph_oracle "py-$layout" --oracle "py-$layout.json" --lang py \ + --gaps benches/graph/known_gaps.toml --corpus "py-$layout" --markdown "graph-layout-$layout.md" + done # Seeded generated Python trees whose generator knows every edge, # definition and reference (tests/support/pygen.rs). grimp and Python @@ -111,8 +130,17 @@ jobs: run: | (cd scripts/graph-oracle && node ../../src/check/oracle/ts.mjs ../../bench-corpus/vite) > ts.json ./target/release/examples/graph_oracle bench-corpus/vite --oracle ts.json --lang ts \ - --max-missing 7 --max-extra 1 --markdown graph-ts.md + --gaps benches/graph/known_gaps.toml --corpus vite --markdown graph-ts.md + + # npm workspaces with tsconfig paths and extends, barrels, .mts/.cts, + # installed as symlinks (scripts/graph-oracle/ts_layouts.mjs). + - name: TypeScript (generated workspace monorepo) + run: | + node scripts/graph-oracle/ts_layouts.mjs ts-workspaces --seed 1 + (cd scripts/graph-oracle && node ../../src/check/oracle/ts.mjs ../../ts-workspaces) > ts-workspaces.json + ./target/release/examples/graph_oracle ts-workspaces --oracle ts-workspaces.json --lang ts \ + --gaps benches/graph/known_gaps.toml --corpus ts-workspaces --markdown graph-ts-workspaces.md - name: Report if: always() - run: cat graph-py.md graph-py-deep.md graph-py-root-init.md graph-ts.md >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true + run: cat graph-py.md graph-py-deep.md graph-py-root-init.md graph-layout-*.md graph-ts.md graph-ts-workspaces.md >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true diff --git a/CHANGELOG.md b/CHANGELOG.md index 90f0d4ec..55b16bae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -162,6 +162,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 made up from the corpus (`--generate-regex`), and `tests/discovery_oracle.rs` compares the indexed files and the FILES box with `rg`. +- **The import graph is checked on monorepo layouts.** Generated + repositories shaped like real ones (`scripts/graph-oracle/layouts.py`, + `ts_layouts.mjs`): Python projects side by side each with its own + `src/`, a root folder that is a package, a namespace package split over + two folders, and npm workspaces with tsconfig `paths` and `extends`, + barrels and `.mts`/`.cts`. grimp and TypeScript give the answer. Every + edge fcs misses or draws wrongly, there and on Django and Vite, is now + a known gap listed with its reason and exact count in + `benches/graph/known_gaps.toml`, instead of a count to stay under. The + Python oracle (`fcs check` included) now builds one graph across + `ROOT` and `ROOT/src`, so imports from one into the other are compared + too. - **Kill and restart.** `examples/chaos.rs` kills the server binary with SIGKILL at random moments (during a build, a load from disk, a stream of edits, a periodic save, a burst of changes, and the save at shutdown), diff --git a/benches/graph/known_gaps.toml b/benches/graph/known_gaps.toml new file mode 100644 index 00000000..dfb4db0c --- /dev/null +++ b/benches/graph/known_gaps.toml @@ -0,0 +1,85 @@ +# Every import edge the graph oracle (examples/graph_oracle.rs) finds fcs +# missing, or drawing wrongly, on each CI corpus, sorted by reason. Each gap +# names the row of docs/book/src/reference/graph-support.md it is, so the +# list is the known limitations, not a number to stay under. +# +# The run fails when an edge fits no gap (a bug, or a limitation to +# document here and on that page) and when a gap's count changes (a fix +# lowers it: lower the count or remove the gap; a rise is a regression). +# +# `spec`, `from` and `to` are regexes on the import text, the importer and +# the target as the report prints them (the corpus folder's name first). +# Generated corpora are written with seed 1 (see the workflow). + +# --- Vite 6.3.5 (TypeScript's resolveModuleName) --- + +[[gap]] +corpus = "vite" +kind = "missing" +spec = '^#' +count = 4 +doc = "package.json `imports` (`#name`)" + +[[gap]] +corpus = "vite" +kind = "missing" +spec = '^\./(bar)?$' +count = 2 +doc = "A folder's own package.json `main`" + +[[gap]] +corpus = "vite" +kind = "missing" +spec = '\.js$' +to = '\.ts$' +count = 1 +doc = "`./x.js` when both x.js and x.ts exist: fcs takes x.js (as Node and bundlers do), TypeScript x.ts" + +[[gap]] +corpus = "vite" +kind = "extra" +to = 'syntax-error\.js$' +count = 1 +doc = "`./x.js` when both x.js and x.ts exist: fcs takes x.js (as Node and bundlers do), TypeScript x.ts" + +# --- Generated: npm workspaces (scripts/graph-oracle/ts_layouts.mjs) --- + +[[gap]] +corpus = "ts-workspaces" +kind = "missing" +spec = '^#' +count = 1 +doc = "package.json `imports` (`#name`)" + +[[gap]] +corpus = "ts-workspaces" +kind = "missing" +spec = '^@acme/ui/theme$' +count = 1 +doc = "`exports` subpath maps (here a package importing itself by name)" + +# --- Generated: Python layouts (scripts/graph-oracle/layouts.py) --- + +[[gap]] +corpus = "py-monorepo" +kind = "missing" +to = '/libs/lib_\w+/src/' +count = 330 +doc = "Other sys.path entries: another project's `src/` in the same repository" + +[[gap]] +corpus = "py-namespace" +kind = "missing" +from = '^py-namespace/src/acme/' +to = '^py-namespace/plugins/src/acme/' +count = 51 +doc = "Other sys.path entries: a namespace package's portion in another folder" + +[[gap]] +corpus = "py-namespace" +kind = "missing" +from = '^py-namespace/plugins/src/acme/' +spec = '^from \.' +to = '^py-namespace/src/acme/' +count = 13 +doc = "Other sys.path entries: a relative import into a namespace package's portion in another folder" diff --git a/docs/book/src/how-it-works/testing.md b/docs/book/src/how-it-works/testing.md index 5cd52d57..a36016ea 100644 --- a/docs/book/src/how-it-works/testing.md +++ b/docs/book/src/how-it-works/testing.md @@ -28,7 +28,7 @@ wrote every line and knows the answer. | `tests/lock_sim.rs` | the engine lock and the watcher's install steps under minutes of made-up load (back-to-back half-second searches, one 20 s search among quick ones, a busy server during a branch switch): every search answered, no search waits past its hold-back plus one step, backlogs drain at half speed or better, a slow search delays only the writer | bounds derived from the protocol, on a simulated clock (`fast_code_search::sim`): the real lock code, the same schedule on every run and OS, four minutes in about a second | `cargo test --test lock_sim` (`-- --ignored --nocapture` prints the tuning table) | | `tests/fault_injection.rs` | failed reads (permission, out of file handles, a file gone between listing and reading), a slow disk, panics while reading, parsing or applying changes, lost changes and lost events, a save onto a full disk or a failed rename: each is retried, reported or repaired, searches are not held up, and the saved index stays whole | expected recovery | `cargo test --features failpoints --test fault_injection` (fault points are listed in `src/faults.rs`) | | `examples/rg_backtest.rs` | fixed and generated queries and regexes on tokio, Django and fixtures, with paging and default limits, on an index built in one go, restarted from disk (`--history reload`) and filled by the watcher (`--history watch`) | `rg` | see [Correctness against ripgrep](../benchmarks/correctness.md) | -| `examples/graph_oracle.rs` | the import graph on Django, Vite and generated deep trees | grimp, TypeScript's resolver | CI job "graph oracle" | +| `examples/graph_oracle.rs` | the import graph on Django, Vite, generated deep trees and generated monorepo layouts (projects side by side with their own `src/`, a root folder that is a package, a split namespace package, npm workspaces with tsconfig `paths` and `extends`); every edge missed or wrong must be a known gap listed with its reason and count | grimp, TypeScript's resolver | CI job "graph oracle", gaps in `benches/graph/known_gaps.toml` | | `examples/ref_oracle.rs`, `examples/impact_oracle.rs` | Python references and diff impact on real repositories | Jedi | CI jobs "ref oracle", "impact oracle" | | `examples/scoreboard.rs` | the server binary over HTTP in seven scenarios (cold build, warm restart, branch switch, ten-minute stream, slow mix, saves under load, event-queue overflow): searches turned away, latency, time until the index matches the disk, memory, CPU, diagnostic captures, crashes, `/api/check` | targets in `benches/scoreboard/targets.toml` | nightly and on pull requests that touch locking or watching (workflow "Scale scoreboard"), see [Scale scoreboard](../benchmarks/scoreboard.md) | | `examples/chaos.rs` | `kill -9` of the server binary at random moments during a build, a load, a stream of edits, a periodic save, a burst and the shutdown save; after each, a restart must load the saved index (never reject it), leave no temp file, catch up with every change and pass `/api/check` | the files on disk | nightly and on pull requests that touch saving or watching (workflow "Scale scoreboard", job "Kill and restart"), see [below](#killed-at-any-moment) | diff --git a/docs/book/src/reference/graph-support.md b/docs/book/src/reference/graph-support.md index f1918302..c54fa4d1 100644 --- a/docs/book/src/reference/graph-support.md +++ b/docs/book/src/reference/graph-support.md @@ -67,7 +67,8 @@ a file `foo.rs` that is not the module. Config read: `Cargo.toml`. | `src/` layouts (`tests/` importing `src/pkg`) | yes: each of those folders' `src/` is tried too | | Absolute imports from a file nested any number of folders deep | yes: every ancestor folder is tried | | A repository whose root folder has an `__init__.py`, run with that folder on `sys.path` (`import app.models`) | yes: when nothing else resolves the import, the outermost package folder is tried as a root if it is the repository root (`.git`) or no folder above it holds an indexed file | -| Other `sys.path` entries (`pyproject.toml`, `setup.cfg`, `PYTHONPATH`) | **no**: no configuration is read, so a package in a sibling folder (`lib/mypkg/` imported from `apps/x/main.py`, with `lib` put on `PYTHONPATH` by your setup) is not found and the import counts as external | +| Other `sys.path` entries (`pyproject.toml`, `setup.cfg`, `PYTHONPATH`) | **no**: no configuration is read, so a package in a sibling folder (`lib/mypkg/` imported from `apps/x/main.py`, with `lib` put on `PYTHONPATH` by your setup) is not found and the import counts as external. This includes a monorepo of projects side by side, each with its own `src/` and installed into one environment (`services/billing` importing `libs/lib_db/src/lib_db`): imports within a project resolve, imports of another project do not | +| Namespace packages (no `__init__.py`) | within one folder, yes; a namespace package split over several folders on `sys.path` (`src/acme/core` and `plugins/src/acme/x`) only where one portion's folder is an ancestor's `src/` of the other, so imports into the other portion, absolute or relative, are mostly missed | | `.pyi` stubs as import targets | **no** | | Standard library, installed packages | no edge, by design | @@ -81,7 +82,7 @@ a file `foo.rs` that is not the module. Config read: `Cargo.toml`. | `import x = require('./x')` (TypeScript) | yes | | Relative specifiers (`./`, `../`, `.`, `..`) | yes | | `@/x`, `~/x` | yes: tsconfig `paths` when it maps them, else from the nearest folder with a `package.json` or `tsconfig.json` (else the repository root), trying `src/x` then `x` | -| Extension probing | the exact file; `.js`/`.jsx` written for a `.ts`/`.tsx` file, `.mjs` for `.mts`, `.cjs` for `.cts`; then `.ts .tsx .js .jsx .mjs .cjs .mts .cts .d.ts .json` appended; then `index` with each of those extensions | +| Extension probing | the exact file; `.js`/`.jsx` written for a `.ts`/`.tsx` file, `.mjs` for `.mts`, `.cjs` for `.cts`; then `.ts .tsx .js .jsx .mjs .cjs .mts .cts .d.ts .json` appended; then `index` with each of those extensions. When both `x.js` and `x.ts` exist, `./x.js` is `x.js` (as Node and bundlers load it), where TypeScript picks `x.ts` | | Imports of stylesheets, `.vue`/`.svelte` components and other assets | yes, when the file is indexed (a bundler resolves them; TypeScript does not) | | A folder's own `package.json` `main` (`import './lib'` where `lib/package.json` names the entry) | **no** | | `package.json` `imports` (`#name`) and a package importing itself by name | **no** | @@ -102,16 +103,30 @@ wrongly: | Language | Oracle | Repository | Result | |---|---|---|---| | Python | [grimp](https://github.com/python-grimp/grimp) 3.17 (`src/check/oracle/py.py`) | Django 5.2 | 3,041 of 3,041 edges, none extra | -| TypeScript, JavaScript | TypeScript 5.9 `ts.resolveModuleName` under each file's tsconfig (`src/check/oracle/ts.mjs`) | Vite 6.3.5 | 99.6% of edges, 1 extra; 4 of the 7 misses are `package.json` `imports` (`#name`); 221 workspace-package edges the oracle cannot see without an install are listed apart | - -CI runs both, and fails if a change misses or adds more edges than that. +| Python | grimp | a package nested 14 levels deep; a repository whose root folder has `__init__.py` (`scripts/graph-oracle/deep_python_tree.py`) | every edge, none extra | +| Python | grimp, with every project's `src/` on `sys.path` | generated: services and libraries side by side, each its own project with `src/` (`layouts.py monorepo`) | 368 of 698 edges, none extra; the 330 missed are imports of another project (other `sys.path` entries, above) | +| Python | grimp | generated: a root folder that is a package, three top-level packages importing each other (`layouts.py root-init`) | 468 of 468 edges, none extra | +| Python | grimp | generated: a namespace package split over `src/` and `plugins/src/` (`layouts.py namespace`) | 98 of 162 edges, none extra; the 64 missed cross from one portion to the other (namespace packages, above) | +| TypeScript, JavaScript | TypeScript 5.9 `ts.resolveModuleName` under each file's tsconfig (`src/check/oracle/ts.mjs`) | Vite 6.3.5 | 1,613 of 1,620 edges, 1 extra: 4 `package.json` `imports` (`#name`), 2 folders with their own `package.json` `main`, and one `./x.js` where `x.ts` exists too (the extra is its other side); 221 workspace-package edges the oracle cannot see without an install are listed apart | +| TypeScript, JavaScript | TypeScript | generated: npm workspaces installed as symlinks, a root `tsconfig.base.json` with `paths` that each project `extends`, barrels, `.mts`/`.cts`, `import type`, `import()`, `require` (`scripts/graph-oracle/ts_layouts.mjs`) | 388 of 390 edges, none extra: one `#name` import and one `exports` subpath | + +Every edge fcs misses or draws wrongly on these is listed, with its reason +and count, in `benches/graph/known_gaps.toml`. CI ("graph oracle") fails +when an edge is not on that list (a bug, or a gap to document here) and +when a listed count changes, so a fix shows up as a gap to lower or +remove. The generated repositories are written with seed 1. ```sh python3 src/check/oracle/py.py bench-corpus/django > py.json cargo run --release --example graph_oracle -- bench-corpus/django --oracle py.json --lang py (cd scripts/graph-oracle && npm ci) # pins TypeScript (cd scripts/graph-oracle && node ../../src/check/oracle/ts.mjs ../../bench-corpus/vite) > ts.json -cargo run --release --example graph_oracle -- bench-corpus/vite --oracle ts.json --lang ts +cargo run --release --example graph_oracle -- bench-corpus/vite --oracle ts.json --lang ts \ + --gaps benches/graph/known_gaps.toml --corpus vite +python3 scripts/graph-oracle/layouts.py monorepo py-monorepo --seed 1 +(set -a; . py-monorepo/oracle.env; set +a; python3 src/check/oracle/py.py py-monorepo) > py-monorepo.json +cargo run --release --example graph_oracle -- py-monorepo --oracle py-monorepo.json --lang py \ + --gaps benches/graph/known_gaps.toml --corpus py-monorepo ``` To run the same comparison on your own code, use `fcs check` (see diff --git a/examples/graph_oracle.rs b/examples/graph_oracle.rs index 31bffb30..26e76a4c 100644 --- a/examples/graph_oracle.rs +++ b/examples/graph_oracle.rs @@ -18,10 +18,16 @@ //! //! An edge the oracle draws and fcs does not is a **miss**; one fcs draws //! and the oracle does not is an **extra** (a wrong edge). Only files fcs -//! indexed count, on both sides. `--max-missing` and `--max-extra` fail the -//! run above a count, so a baseline can only go down. The report groups -//! misses by the shape of the import, which says which gap in -//! `docs/book/src/reference/graph-support.md` each one is. +//! indexed count, on both sides. The report groups misses by the shape of +//! the import. +//! +//! `--gaps benches/graph/known_gaps.toml --corpus NAME` holds the run to +//! the known gaps listed for that corpus, each tied to a row of +//! `docs/book/src/reference/graph-support.md`: every miss and extra must +//! match a listed gap, and each gap must match exactly its `count` edges. +//! A new miss fails the run (a bug, or a gap to document), and so does a +//! gap that shrank (a fix: lower its count or remove it). `--max-missing` +//! and `--max-extra` are the older, count-only check. use anyhow::{Context, Result}; use clap::Parser; @@ -51,6 +57,105 @@ struct Args { /// Write the markdown report here as well as to stderr. #[arg(long)] markdown: Option, + /// Known gaps (TOML); with `--corpus`, every miss and extra must be one. + #[arg(long, requires = "corpus")] + gaps: Option, + /// This tree's name in `--gaps`. + #[arg(long)] + corpus: Option, +} + +#[derive(serde::Deserialize)] +struct GapList { + gap: Vec, +} + +/// A known gap: edges of one corpus fcs misses (or draws wrongly), all +/// for one documented reason. +#[derive(serde::Deserialize)] +struct Gap { + corpus: String, + /// `missing` or `extra`. + kind: String, + /// Regexes on the import text (misses only), the importer and the + /// target, as the report shows them; all given must match. + spec: Option, + from: Option, + to: Option, + /// Exactly this many edges. + count: usize, + /// Why: the row of the graph-support page this is. + doc: String, +} + +struct GapMatcher { + gap: Gap, + spec: Option, + from: Option, + to: Option, + found: usize, +} + +impl GapMatcher { + fn new(gap: Gap) -> Result { + let re = |r: &Option| -> Result> { + r.as_deref() + .map(regex::Regex::new) + .transpose() + .context("a regex in --gaps") + }; + Ok(Self { + spec: re(&gap.spec)?, + from: re(&gap.from)?, + to: re(&gap.to)?, + gap, + found: 0, + }) + } + + fn matches(&self, kind: &str, from: &str, spec: Option<&str>, to: &str) -> bool { + let ok = |re: &Option, text: Option<&str>| { + re.as_ref() + .is_none_or(|re| text.is_some_and(|t| re.is_match(t))) + }; + self.gap.kind == kind + && ok(&self.spec, spec) + && ok(&self.from, Some(from)) + && ok(&self.to, Some(to)) + } +} + +/// Sort `report`'s misses and extras into `gaps`; returns what fits none, +/// and the markdown table of the gaps. +fn check_gaps( + report: &fast_code_search::check::graph::GraphReport, + gaps: &mut [GapMatcher], +) -> (Vec, String) { + let mut unlisted = Vec::new(); + for (from, line, spec, to) in &report.missing { + match gaps + .iter_mut() + .find(|g| g.matches("missing", from, Some(spec), to)) + { + Some(g) => g.found += 1, + None => unlisted.push(format!("missing {from}:{line}: {spec} -> {to}")), + } + } + for (from, to) in &report.extra { + match gaps.iter_mut().find(|g| g.matches("extra", from, None, to)) { + Some(g) => g.found += 1, + None => unlisted.push(format!("extra {from} -> {to}")), + } + } + let mut md = + String::from("\n### Known gaps\n\n| Gap | Kind | Listed | Found |\n|---|---|---:|---:|\n"); + for g in gaps.iter() { + md.push_str(&format!( + "| {} | {} | {} | {} |\n", + g.gap.doc, g.gap.kind, g.gap.count, g.found + )); + } + (unlisted, md) } fn main() -> Result<()> { @@ -74,16 +179,61 @@ fn main() -> Result<()> { started.elapsed().as_secs_f64() ); let report = compare(&corpus.engine, lang, &oracle); - let md = report.markdown( + let mut md = report.markdown( &tree.file_name().unwrap_or_default().to_string_lossy(), args.samples, ); + + let mut failed = Vec::new(); + if let (Some(path), Some(corpus_name)) = (&args.gaps, &args.corpus) { + let list: GapList = toml::from_str( + &std::fs::read_to_string(path) + .with_context(|| format!("reading {}", path.display()))?, + ) + .with_context(|| format!("parsing {}", path.display()))?; + let mut gaps = list + .gap + .into_iter() + .filter(|g| &g.corpus == corpus_name) + .map(GapMatcher::new) + .collect::>>()?; + let (unlisted, table) = check_gaps(&report, &mut gaps); + md.push_str(&table); + if !unlisted.is_empty() { + md.push_str("\n### Not a known gap\n\n```\n"); + for u in &unlisted { + md.push_str(u); + md.push('\n'); + } + md.push_str("```\n"); + failed.push(format!( + "{} edges are not a known gap (a bug, or a gap to list in {})", + unlisted.len(), + path.display() + )); + } + for g in &gaps { + if g.found != g.gap.count { + failed.push(format!( + "{:?} ({}): {} edges, listed as {}{}", + g.gap.doc, + g.gap.kind, + g.found, + g.gap.count, + if g.found < g.gap.count { + " (fixed? lower the count, or remove the gap)" + } else { + "" + } + )); + } + } + } eprint!("{md}"); if let Some(p) = &args.markdown { std::fs::write(p, &md)?; } - let mut failed = Vec::new(); if let Some(m) = args.max_missing.filter(|&m| report.missing.len() > m) { failed.push(format!( "{} missing edges, more than --max-missing {m}", diff --git a/scripts/graph-oracle/layouts.py b/scripts/graph-oracle/layouts.py new file mode 100644 index 00000000..ea58d19f --- /dev/null +++ b/scripts/graph-oracle/layouts.py @@ -0,0 +1,196 @@ +"""Write Python repositories laid out the way real monorepos are, for the +graph oracle (examples/graph_oracle.rs, CI workflow "graph oracle"). + + python scripts/graph-oracle/layouts.py LAYOUT OUT [--seed N] + +Django, the deep tree and the generated codebases (tests/support/pygen.rs) +are each one package under one folder on sys.path. Real repositories are +not, and the graph has broken on their shapes (#170: a root folder with +`__init__.py`). Each layout here is one such shape, filled with seeded +modules that import each other by absolute and relative names: + +- `monorepo`: services and libraries side by side, each its own project + with a `src/` folder (`services/billing/src/billing/...`, + `libs/lib_db/src/lib_db/...`), installed into one environment, so every + `src/` is on sys.path. Services import the libraries by name. +- `root-init`: a repository whose root folder has an `__init__.py` and is + run with the root on sys.path; top-level packages (`apps`, `platform`, + `libs`) import each other as `apps.billing.models`. +- `namespace`: one namespace package (PEP 420, no `__init__.py`) split + over two folders on sys.path (`src/acme/...` and `plugins/src/acme/...`). + +It writes OUT/oracle.env with the settings the oracle (src/check/oracle/ +py.py) needs for the layout: the extra sys.path folders +(FCS_ORACLE_PY_PATHS) and the namespace packages (FCS_ORACLE_PY_NAMESPACES). +OUT/.git is created so the folder looks like a repository's root. +""" + +import os +import random +import sys + +FORMS = ["import", "import_as", "from_module", "from_name", "relative"] + + +class Tree: + def __init__(self, out, rng): + self.out = out + self.rng = rng + # dotted name -> (path of the file, sys.path folder it hangs off) + self.modules = {} + self.order = [] + + def package(self, base, dotted, init=True): + path = os.path.join(self.out, base, *dotted.split(".")) + os.makedirs(path, exist_ok=True) + if init: + self.add(base, dotted, os.path.join(path, "__init__.py")) + + def add(self, base, dotted, path): + self.modules[dotted] = path + self.order.append(dotted) + with open(path, "w") as f: + f.write(f'"""{dotted}."""\n\nNAME_{dotted.replace(".", "_")} = 1\n') + + def module(self, base, dotted): + self.add(base, dotted, os.path.join(self.out, base, *dotted.split(".")) + ".py") + + def imports(self, importer, targets): + """Append imports of `targets` to `importer`'s file.""" + lines = [] + for t in targets: + if t == importer: + continue + form = self.rng.choice(FORMS) + parent, _, leaf = t.rpartition(".") + name = f"NAME_{t.replace('.', '_')}" + if form == "relative" and importer.split(".")[0] == t.split(".")[0]: + rel = relative(importer, t, self.modules[importer].endswith("__init__.py")) + if rel: + lines.append(rel) + continue + if form == "import" or (form == "relative"): + lines.append(f"import {t}") + elif form == "import_as": + lines.append(f"import {t} as m_{t.replace('.', '_')}") + elif form == "from_module" and parent: + lines.append(f"from {parent} import {leaf}") + else: + lines.append(f"from {t} import {name}") + with open(self.modules[importer], "a") as f: + f.write("\n" + "\n".join(lines) + "\n") + + def wire(self, groups, per_module=(1, 4)): + """Each module imports a few from the groups it may see: `groups` + maps a dotted prefix to the prefixes its modules may import.""" + for m in self.order: + allowed = [ + t + for prefix, sees in groups.items() + if m == prefix or m.startswith(prefix + ".") + for t in self.order + if any(t == s or t.startswith(s + ".") for s in sees) + ] + if allowed: + n = self.rng.randint(*per_module) + self.imports(m, self.rng.sample(allowed, min(n, len(allowed)))) + + +def relative(importer, target, importer_is_package): + """`from ..x import y` for `target` seen from `importer`, if expressible.""" + here = importer.split(".") + if not importer_is_package: + here = here[:-1] # the package the module is in + there = target.split(".") + common = 0 + while common < min(len(here), len(there)) and here[common] == there[common]: + common += 1 + if common == 0: + return None + dots = "." * (len(here) - common + 1) + rest = there[common:] + if not rest: + return None + if len(rest) == 1: + return f"from {dots} import {rest[0]}" + return f"from {dots}{'.'.join(rest[:-1])} import {rest[-1]}" + + +def subtree(tree, base, top, rng, depth=3, width=3, modules=4): + """A package `top` under `base`, nested `depth` deep.""" + tree.package(base, top) + frontier = [top] + for _ in range(depth): + nxt = [] + for p in frontier: + for i in range(rng.randint(1, width)): + child = f"{p}.sub{i}" + tree.package(base, child) + nxt.append(child) + for i in range(rng.randint(1, modules)): + tree.module(base, f"{p}.mod{i}") + frontier = nxt + for p in frontier: + for i in range(rng.randint(1, modules)): + tree.module(base, f"{p}.mod{i}") + + +def monorepo(out, rng): + tree = Tree(out, rng) + services = ["billing", "search_svc", "ingest"] + libs = ["lib_db", "lib_auth", "lib_util"] + paths = [] + for kind, names, depth in [("services", services, 3), ("libs", libs, 2)]: + for name in names: + project = os.path.join(out, kind, name) + base = os.path.join(kind, name, "src") + subtree(tree, base, name, rng, depth=depth) + paths.append(base) + with open(os.path.join(project, "pyproject.toml"), "w") as f: + f.write(f'[project]\nname = "{name}"\nversion = "1.0"\n') + groups = {s: [s] + libs for s in services} + groups.update({lib: [lib, "lib_util"] for lib in libs}) + tree.wire(groups) + return {"FCS_ORACLE_PY_PATHS": os.pathsep.join(paths)} + + +def root_init(out, rng): + tree = Tree(out, rng) + with open(os.path.join(out, "__init__.py"), "w") as f: + f.write("") + tops = ["apps", "platform_core", "libs"] + for top in tops: + subtree(tree, "", top, rng) + tree.wire({"apps": tops, "platform_core": ["platform_core", "libs"], "libs": ["libs"]}) + return {} + + +def namespace(out, rng): + tree = Tree(out, rng) + for base, part in [("src", "core"), ("src", "billing"), ("plugins/src", "plugin_x")]: + os.makedirs(os.path.join(out, base, "acme"), exist_ok=True) + subtree(tree, base, f"acme.{part}", rng, depth=2) + tree.wire({"acme": ["acme"]}) + return {"FCS_ORACLE_PY_PATHS": "plugins/src", "FCS_ORACLE_PY_NAMESPACES": "acme"} + + +LAYOUTS = {"monorepo": monorepo, "root-init": root_init, "namespace": namespace} + + +def main(): + args = sys.argv[1:] + seed = 1 + if "--seed" in args: + i = args.index("--seed") + seed = int(args[i + 1]) + del args[i : i + 2] + layout, out = args + os.makedirs(os.path.join(out, ".git"), exist_ok=True) + env = LAYOUTS[layout](out, random.Random(seed)) + with open(os.path.join(out, "oracle.env"), "w") as f: + for k, v in env.items(): + f.write(f"{k}={v}\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/graph-oracle/ts_layouts.mjs b/scripts/graph-oracle/ts_layouts.mjs new file mode 100644 index 00000000..efad4962 --- /dev/null +++ b/scripts/graph-oracle/ts_layouts.mjs @@ -0,0 +1,215 @@ +// Write a TypeScript workspace monorepo laid out the way real ones are, for +// the graph oracle (examples/graph_oracle.rs, CI workflow "graph oracle"). +// +// node scripts/graph-oracle/ts_layouts.mjs OUT [--seed N] +// +// Vite is one such repository, pinned; this one is generated, so every +// shape below is there whatever Vite does next: +// +// - npm workspaces (`packages/*`, `apps/*`), each package with a +// `package.json` name (`@acme/ui`) and installed into `node_modules` as +// a symlink, as `npm install` leaves it; +// - a root `tsconfig.base.json` with `baseUrl` and `paths` (`@acme/x`, +// `@acme/x/*`), which each project's `tsconfig.json` `extends`, and an +// app adding its own `paths` alias (`~/x`); +// - barrels (`export * from './a'`, `export { b } from './b'`), `import +// type`, dynamic `import()`, `.mts`/`.cts` files imported as `.mjs`/ +// `.cjs`, `require` in CommonJS, JSON imports, folder `index` files; +// - forms fcs does not handle, on purpose, so they stay listed gaps: +// `package.json` `imports` (`#internal/x`), `exports` subpaths +// (`@acme/ui/theme`), and a folder whose own `package.json` names its +// entry (`import './legacy'`). +// +// Modules import each other at random (seeded), within a package by +// relative path and across packages by name. + +import fs from "node:fs"; +import path from "node:path"; + +const args = process.argv.slice(2); +let seed = 1; +const si = args.indexOf("--seed"); +if (si >= 0) { + seed = Number(args[si + 1]); + args.splice(si, 2); +} +const out = path.resolve(args[0]); + +// Mulberry32: small and the same on every platform. +let state = seed >>> 0; +function rand() { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; +} +const below = (n) => Math.floor(rand() * n); +const pick = (xs) => xs[below(xs.length)]; + +function write(file, text) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, text); +} +const json = (v) => JSON.stringify(v, null, 2) + "\n"; + +fs.rmSync(out, { recursive: true, force: true }); +fs.mkdirSync(path.join(out, ".git"), { recursive: true }); + +const packages = ["ui", "data", "utils", "auth", "billing"]; +const apps = ["web", "api"]; +write( + path.join(out, "package.json"), + json({ name: "acme", private: true, workspaces: ["packages/*", "apps/*"] }), +); +const paths = {}; +for (const p of packages) { + paths[`@acme/${p}`] = [`packages/${p}/src`]; + paths[`@acme/${p}/*`] = [`packages/${p}/src/*`]; +} +write( + path.join(out, "tsconfig.base.json"), + json({ + compilerOptions: { + target: "es2022", + module: "esnext", + moduleResolution: "bundler", + allowJs: true, + resolveJsonModule: true, + baseUrl: ".", + paths, + }, + }), +); + +// Modules per project: { project: [{ file, spec(fromFile) }] } +const modules = {}; +function addModule(project, rel, body = "") { + const file = path.join(out, project, rel); + (modules[project] ??= []).push(file); + write(file, `// ${project}/${rel}\nexport const v_${modules[project].length} = 1;\n${body}`); + return file; +} + +function relSpec(from, to) { + let r = path.relative(path.dirname(from), to).split(path.sep).join("/"); + if (!r.startsWith(".")) r = "./" + r; + // Written the way TypeScript code imports them. + r = r.replace(/\.mts$/, ".mjs").replace(/\.cts$/, ".cjs"); + if (rand() < 0.5) r = r.replace(/\.(ts|tsx)$/, ""); + else r = r.replace(/\.(ts|tsx)$/, ".js"); + return r.replace(/\/index(\.js)?$/, ""); +} + +for (const p of packages) { + const dir = `packages/${p}`; + write( + path.join(out, dir, "package.json"), + json({ + name: `@acme/${p}`, + version: "1.0.0", + main: "src/index.ts", + types: "src/index.ts", + ...(p === "ui" + ? { exports: { ".": "./src/index.ts", "./theme": "./src/theme/main.ts" } } + : {}), + ...(p === "data" ? { imports: { "#internal/*": "./src/internal/*.ts" } } : {}), + }), + ); + write(path.join(out, dir, "tsconfig.json"), json({ extends: "../../tsconfig.base.json" })); + for (let i = 0; i < 20 + below(20); i++) { + const sub = pick(["", "lib/", "components/", "lib/deep/"]); + const ext = pick([".ts", ".ts", ".tsx", ".mts", ".cts", ".js"]); + addModule(dir, `src/${sub}m${i}${ext}`); + } + addModule(dir, `src/lib/index.ts`); + if (p === "ui") addModule(dir, "src/theme/main.ts"); + if (p === "data") addModule(dir, "src/internal/cache.ts"); + write(path.join(out, dir, "src/config.json"), json({ name: p })); +} +for (const a of apps) { + const dir = `apps/${a}`; + write(path.join(out, dir, "package.json"), json({ name: `@acme/app-${a}`, private: true })); + write( + path.join(out, dir, "tsconfig.json"), + json({ + extends: "../../tsconfig.base.json", + compilerOptions: { paths: { ...paths, "~/*": [`apps/${a}/src/*`] } }, + }), + ); + for (let i = 0; i < 30 + below(20); i++) { + const sub = pick(["", "pages/", "routes/", "lib/"]); + const ext = a === "web" ? pick([".tsx", ".ts"]) : pick([".ts", ".mts", ".cts"]); + addModule(dir, `src/${sub}m${i}${ext}`); + } + // A folder whose own package.json names its entry. + write(path.join(out, dir, "src/legacy/package.json"), json({ main: "./entry.js" })); + addModule(dir, "src/legacy/entry.js"); +} + +// Installed workspaces, as npm leaves them. +for (const p of packages) { + const link = path.join(out, "node_modules/@acme", p); + fs.mkdirSync(path.dirname(link), { recursive: true }); + fs.symlinkSync(path.join("..", "..", "packages", p), link, "dir"); +} + +// Barrels: each package's index re-exports some of its modules. +for (const p of packages) { + const dir = `packages/${p}`; + const index = path.join(out, dir, "src/index.ts"); + const own = modules[dir].filter((f) => !/\.cts$/.test(f)); + const lines = []; + for (const f of own.slice(0, 4)) { + const spec = relSpec(index, f); + lines.push(rand() < 0.5 ? `export * from '${spec}';` : `export { v_1 as x${lines.length} } from '${spec}';`); + } + write(index, lines.join("\n") + "\n"); +} + +function importLine(from, spec, n) { + const cjs = /\.cts$/.test(from); + if (cjs) return `const r${n} = require('${spec}');`; + const forms = [ + `import { v_1 as i${n} } from '${spec}';`, + `import * as i${n} from '${spec}';`, + `const i${n} = await import('${spec}');`, + `export * from '${spec}';`, + ]; + if (/\.(ts|tsx|mts)$/.test(from)) forms.push(`import type { T${n} } from '${spec}';`); + return pick(forms); +} + +let n = 0; +for (const [project, files] of Object.entries(modules)) { + const isApp = project.startsWith("apps/"); + for (const from of files) { + const lines = []; + for (let k = 0; k < 1 + below(4); k++) { + const choice = below(10); + if (choice < 5) { + const to = pick(files); + if (to !== from) lines.push(importLine(from, relSpec(from, to), n++)); + } else if (choice < 8) { + const p = pick(packages); + const spec = pick([`@acme/${p}`, `@acme/${p}/lib`, `@acme/${p}/config.json`]); + lines.push(importLine(from, spec, n++)); + } else if (isApp && choice === 8) { + const to = pick(files); + const rel = path.relative(path.join(out, project, "src"), to).split(path.sep).join("/"); + lines.push(importLine(from, "~/" + rel.replace(/\.(ts|tsx)$/, ""), n++)); + } else if (choice === 9) { + // The forms fcs does not handle, now and then. + const odd = pick([ + isApp ? "./legacy" : null, + project === "packages/data" ? "#internal/cache" : null, + "@acme/ui/theme", + ]); + if (odd && !(odd === "./legacy" && path.dirname(from) !== path.join(out, project, "src"))) { + lines.push(importLine(from, odd, n++)); + } + } + } + fs.appendFileSync(from, lines.join("\n") + "\n"); + } +} diff --git a/src/check/oracle/py.py b/src/check/oracle/py.py index ce5032bd..146e4d61 100644 --- a/src/check/oracle/py.py +++ b/src/check/oracle/py.py @@ -8,6 +8,11 @@ on sys.path. External packages are left out: fcs never indexes them either. Output: {"files": [...], "edges": [{from, line, spec, to}]}, absolute paths, one edge per import line; "files" are the modules analysed. + +For layouts with more on sys.path (scripts/graph-oracle/layouts.py): +FCS_ORACLE_PY_PATHS adds folders under ROOT (os.pathsep-separated), and +FCS_ORACLE_PY_NAMESPACES names namespace packages (folders without +`__init__.py`, possibly split over several of those folders). """ import json @@ -17,44 +22,61 @@ import grimp -def packages(base): +def packages(base, namespaces): return sorted( name for name in os.listdir(base) if name.isidentifier() - and os.path.isfile(os.path.join(base, name, "__init__.py")) + and ( + os.path.isfile(os.path.join(base, name, "__init__.py")) + or (name in namespaces and os.path.isdir(os.path.join(base, name))) + ) ) -def module_file(base, module): +def module_file(bases, module): parts = module.split(".") - as_package = os.path.join(base, *parts, "__init__.py") - if os.path.isfile(as_package): - return as_package - as_module = os.path.join(base, *parts) + ".py" - return as_module if os.path.isfile(as_module) else None + for base in bases: + as_package = os.path.join(base, *parts, "__init__.py") + if os.path.isfile(as_package): + return as_package + as_module = os.path.join(base, *parts) + ".py" + if os.path.isfile(as_module): + return as_module + return None def main(): root = os.path.realpath( sys.argv[1] if len(sys.argv) > 1 else os.environ.get("FCS_ORACLE_ROOT", ".") ) + extra = [p for p in os.environ.get("FCS_ORACLE_PY_PATHS", "").split(os.pathsep) if p] + namespaces = set( + n for n in os.environ.get("FCS_ORACLE_PY_NAMESPACES", "").split(",") if n + ) + bases = [] + for b in [root, os.path.join(root, "src")] + [os.path.join(root, p) for p in extra]: + b = os.path.realpath(b) + if os.path.isdir(b) and b not in bases: + bases.append(b) + # The first folder holding a top-level name wins, as on sys.path; + # a namespace package collects every folder's portion. + pkgs = [] + for base in bases: + for name in packages(base, namespaces): + if name not in pkgs: + pkgs.append(name) files, edges = [], [] - for base in (root, os.path.join(root, "src")): - if not os.path.isdir(base): - continue - pkgs = packages(base) - if not pkgs: - continue - sys.path.insert(0, base) + if pkgs: + sys.path[:0] = bases graph = grimp.build_graph(*pkgs, include_external_packages=False, cache_dir=None) for importer in sorted(graph.modules): - src = module_file(base, importer) + src = module_file(bases, importer) if src is None: continue files.append(src) for imported in sorted(graph.find_modules_directly_imported_by(importer)): - dst = module_file(base, imported) + dst = module_file(bases, imported) if dst is None: continue for d in graph.get_import_details(importer=importer, imported=imported):