diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index c4c41cb..ddcbeb7 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ { "name": "brief2ship", "description": "Search and score existing code before greenfield AI builds, then ship with proof", - "version": "0.6.2", + "version": "0.7.0", "source": "./", "author": { "name": "Michael Costea", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index b2e4983..ddae6c4 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "brief2ship", "description": "Repo-first discovery and verified delivery for AI coding agents", - "version": "0.6.2", + "version": "0.7.0", "author": { "name": "Michael Costea", "url": "https://michaelcostea.com/" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f5fa89..325a5bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,11 +25,23 @@ jobs: python-version: ${{ matrix.python }} cache: pip - name: Install build tools - run: python -m pip install --upgrade pip build + run: python -m pip install build==1.6.0 ruff==0.16.6 pyright==1.1.411 + - name: Critical lint + run: ruff check --select E9,F63,F7,F82 src scripts tests + - name: Type checks + run: pyright - name: Validate release contract run: python scripts/validate-release.py + - name: Task-quality benchmark report + run: python scripts/benchmark-discovery.py --output quality-report.json - name: Build wheel and source archive run: python -m build + - name: Test extracted source archive + run: >- + python -c "import glob,subprocess,sys; + archives=glob.glob('dist/*.tar.gz'); + assert len(archives)==1, archives; + subprocess.check_call([sys.executable,'scripts/check-sdist.py',archives[0]])" - name: Install built wheel run: >- python -c "import glob,subprocess,sys; @@ -42,6 +54,7 @@ jobs: python -m brief2ship --version brief2ship discover --help python -c "from brief2ship import DiscoveryConfig, discover; assert DiscoveryConfig and discover" + python scripts/smoke-installed.py optional-extractor: name: Optional Trafilatura adapter diff --git a/CHANGELOG.md b/CHANGELOG.md index 68931c4..99ba5a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 0.7.0 - 2026-09-05 + +- separate discovery health, decision status, and reuse disposition; provider failures, empty retrieval and missing inspections now yield `inconclusive` with CLI exit 5, never a false clean-build recommendation +- decide against the full evaluated set; make `--limit` presentation-only and preserve every evaluated candidate, inspection allocation and selected identity in v2 receipts +- add deterministic core/constraint query planning, bounded GitHub reciprocal-rank fusion, registry core queries and source ranks without model calls or new runtime dependencies +- gate ranking on relevance, use confidence-adjusted decision scores and prefer ready evidence; retain original constraints as unverified checks +- preserve unknown dependency metadata and package/repository evidence scope instead of silently awarding dependency-free credit +- recognize canonical full-text MIT bodies while preserving raw evidence; free-form copyright notices are review-required rather than granted authority through a regex +- add explicit `--summary` JSON output with decision state, source health, counts, score, blockers and receipt paths +- add a synthetic task-quality regression benchmark, pinned lint/type checks, clean installed-wheel smoke and extracted-sdist validation in CI +- document v2 migration and native PowerShell onboarding; update versioned installation routes to v0.7.0 while keeping historical demo evidence pinned + ## 0.6.2 - 2026-09-05 - reposition Brief2Ship around its sharpest searchable job: stop AI coding agents rebuilding what already exists diff --git a/MANIFEST.in b/MANIFEST.in index b5b280e..bb4d198 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,12 +6,13 @@ include CONTRIBUTING.md include CODE_OF_CONDUCT.md include SECURITY.md include ROADMAP.md +include pyrightconfig.json include docs/*.md recursive-include docs/assets *.gif *.png *.svg *.txt recursive-include docs/plans *.md recursive-include skills *.md recursive-include templates *.md recursive-include scripts *.py -recursive-include tests *.py +recursive-include tests *.py *.json recursive-include .github *.md *.yml recursive-include .claude-plugin *.json diff --git a/README.md b/README.md index 289b336..72fe36b 100644 --- a/README.md +++ b/README.md @@ -12,14 +12,27 @@ Brief2Ship is a dependency-free Python CLI and portable Agent Skill for **repo-f ![Terminal demo showing Brief2Ship selecting the local repository for reuse after evidence scoring](https://raw.githubusercontent.com/five0nit/brief2ship/v0.6.2/docs/assets/brief2ship-demo.gif) -*Real dogfood receipt: `local/brief2ship`, 70.50/100, inspected, `selective-reuse`. [Read the source transcript](https://github.com/five0nit/brief2ship/blob/v0.6.2/docs/assets/demo-transcript.txt) or [see more decision cases](https://github.com/five0nit/brief2ship/blob/v0.6.2/docs/case-studies.md).* +*Historical v0.6.2 dogfood receipt (not a v0.7.0 result): `local/brief2ship`, 70.50/100, inspected, `selective-reuse`. [Read the source transcript](https://github.com/five0nit/brief2ship/blob/v0.6.2/docs/assets/demo-transcript.txt) or [see more decision cases](https://github.com/five0nit/brief2ship/blob/v0.6.2/docs/case-studies.md).* + +## v0.7.0 — reliable reuse decisions + +This release adds reliable reuse decisions: explicit `inconclusive` outcomes, +intent-preserving queries, complete evidence receipts, and `--summary` JSON output. +Discovery receipt and scoring contracts are now v2. Read the +[migration and decision guide](docs/code-discovery.md) before updating automation; +exit `5` means insufficient evidence, not permission to build from scratch. +For native Windows setup, see [PowerShell onboarding](docs/windows-powershell.md). + +Install **v0.7.0** using the tagged commands below. The demo above is explicitly +pinned to v0.6.2 historical evidence; its result is not a v0.7.0 claim. +PyPI publication remains separate from the GitHub release. ## 30-second quickstart Run the tagged release without a permanent install: ```bash -uvx --from git+https://github.com/five0nit/brief2ship.git@v0.6.2 \ +uvx --from git+https://github.com/five0nit/brief2ship.git@v0.7.0 \ brief2ship discover "local robots-aware web scraper" \ --sources github,pypi,npm,crates,huggingface \ --limit 5 --inspect-top 2 \ @@ -29,7 +42,7 @@ uvx --from git+https://github.com/five0nit/brief2ship.git@v0.6.2 \ Add a bounded local workspace to the same comparison: ```bash -uvx --from git+https://github.com/five0nit/brief2ship.git@v0.6.2 \ +uvx --from git+https://github.com/five0nit/brief2ship.git@v0.7.0 \ brief2ship discover "local robots-aware web scraper" \ --local /path/to/scoped/workspace \ --sources local,github,pypi,npm,crates,huggingface \ @@ -47,7 +60,8 @@ discovery/ └── worktrees/ # bounded inspection clones when required ``` -The result records one explicit disposition: +Conclusive discovery records one explicit disposition (inconclusive runs record +`inconclusive` when evidence cannot support a decision): - `use-as-library` - `fork` @@ -95,7 +109,7 @@ Brief2Ship inspects manifests, dependency declarations, licenses, tests, CI, doc ### Versioned GitHub release ```bash -uv tool install git+https://github.com/five0nit/brief2ship.git@v0.6.2 +uv tool install git+https://github.com/five0nit/brief2ship.git@v0.7.0 brief2ship doctor ``` @@ -141,13 +155,13 @@ Brief2Ship follows the open [Agent Skills specification](https://agentskills.io/ GitHub CLI 2.90 or newer can install the tagged skill for supported agent hosts: ```bash -gh skill install five0nit/brief2ship brief2ship@v0.6.2 +gh skill install five0nit/brief2ship brief2ship@v0.7.0 ``` Example host-specific installation: ```bash -gh skill install five0nit/brief2ship brief2ship@v0.6.2 \ +gh skill install five0nit/brief2ship brief2ship@v0.7.0 \ --agent claude-code --scope user ``` @@ -172,7 +186,7 @@ Default profile: ```bash hermes skills install \ - https://raw.githubusercontent.com/five0nit/brief2ship/v0.6.2/skills/brief2ship/SKILL.md \ + https://raw.githubusercontent.com/five0nit/brief2ship/v0.7.0/skills/brief2ship/SKILL.md \ --category software-development \ --yes ``` @@ -182,7 +196,7 @@ Named profile: ```bash hermes profile create qa --no-alias --no-skills hermes --profile qa skills install \ - https://raw.githubusercontent.com/five0nit/brief2ship/v0.6.2/skills/brief2ship/SKILL.md \ + https://raw.githubusercontent.com/five0nit/brief2ship/v0.7.0/skills/brief2ship/SKILL.md \ --category software-development \ --yes hermes --profile qa skills list @@ -252,16 +266,21 @@ See [docs/free-scraping.md](docs/free-scraping.md) and [docs/code-discovery.md]( ## Proof and trust -The v0.6.1 compatibility release passed: - -- 141 tests on Python 3.11, 3.12, and 3.13; -- seven Linux/Windows GitHub Actions jobs; -- Ruff critical selectors; -- Pyright with 0 errors and 0 warnings; -- clean wheel and extracted-sdist tests; -- live HTTPS positive and hostname-mismatch negative controls. - -Current source-checkout evidence is retained at `docs/releases/v0.6.2-discoverability-receipt.md`. Release receipts remain outside built packages so their artifact hashes cannot become self-referential. Historical v0.6.1 evidence remains at `docs/releases/v0.6.1-single-search-skill-receipt.md`. +The v0.7.0 implementation passed local verification: + +- 191 tests on Python 3.11, 3.12, and 3.13; +- native Windows validation, with four platform-specific skips; +- 24/24 synthetic decision-regression cases (not real-world accuracy claims); +- Ruff critical selectors and Pyright with 0 errors and 0 warnings; +- clean installed-wheel and extracted-sdist checks; +- independent review and a live five-source discovery probe. + +GitHub Actions runs the Linux/Windows matrix and optional Trafilatura adapter on +pull requests and `main`; inspect the CI badge for the current hosted result. +Historical pre-publication evidence is retained at `docs/releases/v0.7.0-local-verification.md`. +Release receipts remain outside built packages so their artifact hashes cannot become self-referential. +Earlier evidence remains at `docs/releases/v0.6.2-discoverability-receipt.md` and +`docs/releases/v0.6.1-single-search-skill-receipt.md`. ## Repository map diff --git a/ROADMAP.md b/ROADMAP.md index 5f56c0a..917f61d 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -10,13 +10,23 @@ Brief2Ship stays deliberately lean. Roadmap items must improve repo-first decisi - improve first-run examples, error messages, and receipt readability; - publish small, reproducible decision cases rather than unsupported savings claims. -## Next — v0.7 evidence quality +## v0.7 evidence quality: reliable reuse decisions -- normalize common package-license metadata without weakening hard blockers; -- improve multi-query recall while keeping provider limits deterministic; -- make inspection-budget allocation easier to explain from receipts; -- add fixture-backed examples for local/public deduplication and partial provider failure; -- expose concise summary output without removing the full Markdown/JSON receipt. +- explicit inconclusive decisions for incomplete collection or insufficient evidence; +- display-only limits, complete evaluated/inspection receipts and selected identities; +- bounded core/constraint query planning and relevance/confidence-aware ranking; +- unknown dependency evidence, package/repository scoping and strict full-text MIT normalization; +- concise JSON summary alongside full v2 receipts; +- synthetic task-quality regressions plus automated lint/type/wheel/sdist gates. + +Included in v0.7.0. PyPI publication and broader host validation remain separate work. + +## Next evidence-quality work + +- human-reviewed real-world briefs and provider recall benchmarks, beyond synthetic regression fixtures; +- package-subtree identification for monorepos before claiming package-specific test readiness; +- additional canonical license formats with conservative, fixture-backed normalization; +- better source-budget fairness and explainable query recall without mandatory model calls. ## Later — only with demonstrated demand diff --git a/VERSION b/VERSION index b616048..faef31a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.2 +0.7.0 diff --git a/docs/code-discovery.md b/docs/code-discovery.md index 9587787..71cecc9 100644 --- a/docs/code-discovery.md +++ b/docs/code-discovery.md @@ -1,161 +1,199 @@ -# Code and Solution Discovery +# Code discovery and v2 migration -Brief2Ship v0.6 is the single repo-search contract. It searches scoped local workspaces plus public code/package ecosystems before greenfield implementation, normalizes candidates, checks evidence, optionally inspects a bounded set, reranks, and produces JSON/Markdown receipts. +Brief2Ship searches a **bounded, selected source scope**. It does not prove that +no reusable software exists. Normal discovery reads metadata and source; it +never installs candidate dependencies or executes candidate code. -## Quick start +## Run ```bash -brief2ship discover "local web scraper with robots support" \ +brief2ship discover "local robots-aware web scraper" \ --local /path/to/scoped/workspace \ --sources local,github,pypi,npm,crates,huggingface \ - --per-source 10 \ - --limit 10 \ - --total-timeout 180 \ - --inspect-top 3 \ - --output discovery/ + --per-source 5 --limit 5 --inspect-top 2 --total-timeout 180 \ + --summary --output discovery/ ``` -The output directory must be new or empty. Results include: +The output directory must be new or empty. Omit `--summary` to preserve the +existing stdout form: one Markdown receipt path. `--summary` emits a JSON object, +not a replacement for the full receipts. Both forms use identical exit semantics. + +## Decision contract + +The v2 receipt separates: + +- `discovery_status`: source collection health (`complete`, `partial`, `failed`). + Complete means requested providers returned successfully, **not exhaustive recall**. +- `decision_status`: `complete`, `provisional`, or `inconclusive`. +- `overall_recommendation`: the supported reuse disposition, `build-clean`, or + `inconclusive` when the run cannot support an actionable disposition. +- `selected_candidate_id`: stable identity of the selected, statically inspected + candidate, or null. A top-ranked lead is not necessarily selected. +- `incomplete_reasons`: why a supported decision could not be made. + +`inconclusive` is an evidence state, not a sixth reuse strategy. Failed or partial +provider collection, an empty result set, failed inspection or an uninspected +lead cannot silently become `build-clean`. Increase a bounded inspection budget, +repair a failed provider, narrow the scope explicitly or refine the brief; +do not automatically authorize greenfield work on exit 5. + +An inspected reusable candidate may be selected **provisionally**. Required +checks, unknown constraint matches and test/OSV gaps remain in its evidence. +Provisional selection is a lead for further validation, not proof of production +readiness. A supported clean-build conclusion applies only to the evaluated set +and recorded policy, never the entire ecosystem. + +Exit codes for `discover`: + +| Code | Meaning | +|---:|---| +| 0 | Supported complete or provisional decision; inspect `decision_status` and required checks | +| 2 | Invalid command-line arguments | +| 3 | Safety policy blocked the operation | +| 4 | Operational/input/output failure | +| 5 | Receipts written, but evidence is insufficient for a decision | + +This changes v1, which returned success whenever candidates existed and could +turn source outages or display truncation into `build-clean`. + +## Queries and evidence + +`query_plan` preserves the original brief, extracted core query, explicit +constraints and up to three deterministic variants. The parser removes only +recognized instruction/constraint patterns; it is not an LLM or a semantic +requirements solver. `web scraper` survives the brief +`local robots-aware web scraper`; `local` and `robots-aware` remain unverified +checks rather than disappearing or being claimed as satisfied. + +GitHub searches at most three variants and uses reciprocal-rank fusion +(`1 / (60 + source rank)`), deduplicating before the per-source candidate cap. +Other registries search the core query under their existing bounded provider +contracts. Each source receipt records attempted queries, endpoints, status, +errors and warnings. Source ranks are tie-breakers, never raw cross-registry +relevance comparisons. Later providers still share the total wall-clock budget; +source failures and truncation must remain visible. + +Missing dependency metadata stays unknown. PyPI absent/null/malformed declarations +are not an observed empty list. For npm, an absent `dependencies` field in an +actually retrieved version document is contract-confirmed zero; a missing version +document, failed hydration or malformed dependency field is unknown. Scores do +not credit an unknown count as verified dependency-free. + +Package identity remains separate from repository evidence. `repository_evidence` +records repository-level metadata merged into package leads. Repository-wide +license, descriptions, dates and dependencies do not overwrite unknown or +package-specific observations. `inspection` covers the whole repository, not a +proven package subtree; package-level readiness retains an explicit scope check. + +Raw `license` evidence remains unchanged. `normalized_license` accepts supported +metadata identifiers. `license_kind=file` requires a complete canonical grant; +a title-only file is not treated as a metadata license identifier. Inspection +retains the raw file and records a superseded repository metadata license in +`repository_evidence.prior_metadata_license` when the observations differ. +The normalizer accepts supported identifiers and exact canonical MIT grant text +with whitespace and exact title variations. **Free-form copyright notices are +recognized, not automatically approved**: `license_body_match=MIT` records a +complete body, but `normalized_license=null` and `license_review_required=true` +prevent reuse authorization. Arbitrary holder names and hidden conditions cannot +be distinguished reliably by a regex; there is no holder allowlist or vocabulary +denylist. This deliberately requires review for ordinary copyright-bearing MIT +files too. Modified/truncated grants and unknown identifiers remain blocked from +automatic reuse. This is a conservative reuse filter, not legal +advice or a complete SPDX expression parser. + +## Scoring contract and ordering + +The existing raw `/100` component sum remains available as `score.total`: + +| Component | Maximum | +|---|---:| +| Core feature match / relevance | 25 | +| Maintenance/activity | 15 | +| Dependency weight | 10 | +| Security/license/OSV posture | 15 | +| Test quality and CI | 10 | +| Portability | 10 | +| Reuse readiness/adaptation | 10 | +| Adoption/issue health | 5 | + +`score.decision_score = max(0, total - unknown_cost)` discounts missing evidence. +Ranking first gates on feature relevance (at least 8/25), then usable evidence, +ready/provisional status and decision score. Within-source rank is a stable, +bounded tie-breaker. A popular unrelated project cannot beat a relevant reuse +lead merely by accumulating maintenance and adoption points. + +Candidate disposition thresholds retain the raw component contract. Run-level +selection additionally requires static inspection and successful source +collection. `recommendation_status`, `required_checks`, `hard_blockers`, coverage +and per-component evidence must be read alongside the score. + +## Repository inspection and complete receipts ```text discovery/ -├── discovery.json -├── discovery.md -├── candidates/ -│ ├── 01-....json -│ └── ... -└── worktrees/ # bounded public clones only; local candidates stay in place +├── discovery.md # bounded table plus complete candidate/inspection evidence +├── discovery.json # full reconstruction of the bounded evaluation +├── candidates/ # one JSON file per evaluated candidate, not only display rows +└── worktrees/ # bounded inspection clones when needed ``` -## Sources +- `candidates` remains the display shortlist, bounded by `--limit`. +- `evaluated_candidates` contains every deduplicated, scored candidate. +- `inspection_decisions` records allocation reasons and results for attempted + inspections. An inspected candidate falling outside the table does not vanish. +- The supported selected candidate is pinned into the display shortlist. +- Changing `--limit` cannot change the build/reuse decision. +- Markdown escapes untrusted candidate text and terminal/bidirectional controls; + JSON preserves raw evidence using JSON encoding. -### Local workspaces +The version markers change from `brief2ship-discovery-v1` and `brief2ship-score-v1` +to `brief2ship-discovery-v2` and `brief2ship-score-v2`. +The sandbox policy remains `brief2ship-bwrap-v2`. Consumers should reject unknown +schemas rather than inferring success from a nonempty `candidates` array. -Use repeatable `--local PATH` arguments. Supplying one automatically enables the `local` source; use `--sources local` for a fully offline local-only pass. - -Local discovery is read-only and bounded to five roots, 10,000 visited directories, 10,000 entries and 2,000 retained filenames per directory, depth 16, 500 candidate projects, and the shared `--total-timeout`. Roots are traversed round-robin and breadth-first so a broad root or deep subtree cannot starve another root's shallow projects. Hidden/VCS metadata, dependency/vendor trees, build outputs, caches, and symlinked directories are skipped. Project identity comes from real manifests or a Git checkout; README text alone does not turn every nested folder into a project. Credential-bearing remotes are rejected, remote query/fragment data is stripped, and symlinked `.git` metadata is never followed. Files are opened no-follow and read through a bounded descriptor. Receipts retain local paths, so review them before public sharing. - -`--inspect-top` statically inspects selected local projects in place. It never executes them unless the separate explicit sandbox-test gate is enabled. - -### GitHub - -Uses GitHub's REST repository-search API and repository/community-profile metadata. If a long-form target sentence returns zero repositories, Brief2Ship performs one deterministic, receipt-visible fallback using up to three non-generic terms; it never launches an unbounded query fan-out. Authentication is optional: +## Regression benchmark and release gates ```bash -export GH_TOKEN=... -brief2ship discover "..." --sources github --output discovery/ +python scripts/benchmark-discovery.py --output quality-report.json +python scripts/validate-release.py ``` -`GITHUB_TOKEN` is also accepted. Tokens are sent only to `api.github.com`, are never rendered, and only improve rate limits. Discovery API redirects are rejected so bearer headers cannot cross hosts. Private results are filtered even when an authenticated token can see them. Unauthenticated search remains supported but has a low rate limit. - -### PyPI - -PyPI's HTML search is currently protected by a JavaScript client challenge. Brief2Ship does not bypass it. Instead, it downloads the official `/simple/` package-name index, applies deterministic local package-name token matching, caches names for seven days, then fetches official package JSON for the best matches. Because the Simple index contains names rather than descriptions, PyPI discovery can miss packages whose relevance appears only in metadata text. - -The initial Simple index is approximately 42 MB and has a hard 50 MB response cap. Use `--refresh-cache` to replace the cached name list. - -### npm - -Uses the public npm registry search endpoint and package documents. It records version, repository, license, update date, relevance, declared runtime dependency count, and separate test/development signals. - -### crates.io - -Uses the public crate search and exact-version dependency endpoints with a descriptive Brief2Ship User-Agent and a one-request-per-second host throttle. It records downloads, activity, license, repository, version, and dependency count. - -### Hugging Face - -Searches public models, datasets, and Spaces. It records artifact kind, tags, library/pipeline metadata, license, downloads, likes, and last modification time. Hugging Face artifacts without canonical GitHub repositories can be scored but are not cloned by the GitHub inspector. - -### OSV - -For exact PyPI, npm, and crates.io package versions, Brief2Ship queries the free OSV API. Finding IDs are preserved. A failed or unsupported OSV query remains `unknown`; it is never silently treated as clean. - -## Scoring contract - -Scores are deterministic for the same evidence and clock: - -| Dimension | Maximum | Main evidence | -|---|---:|---| -| Feature match | 25 | query-token coverage in name, description, topics, and inspected manifests | -| Maintenance/activity | 15 | last update/publish age; archived projects score zero | -| Dependency weight | 10 | declared dependency count; lower is easier to audit/reuse | -| Security posture | 15 | OSV result, declared license, security policy, archived state | -| Test quality | 10 | test files, CI, detected test command, sandbox result | -| Portability | 10 | ecosystem/language and explicit cross-platform/restriction signals | -| Reuse readiness | 10 | repository, description, license, docs, examples, manifests, source footprint | -| Adoption/health | 5 | stars/downloads plus forks, real watchers, contributor depth, and true issue-only ratio penalty after inspection | - -Unknown data receives an explicit partial-neutral component value, plus separate weighted evidence coverage and unknown-cost fields. It never becomes silent positive evidence. Every component contains a human-readable evidence list in JSON and Markdown. JSON records `brief2ship-discovery-v1`, `brief2ship-score-v1`, the candidate's package version or inspected commit identity, recommendation status, hard blockers, and required checks. +The benchmark uses committed **fictional, agent-authored synthetic fixtures**: +short and constraint-rich briefs, non-obvious names, popular unrelated projects, +blocked lexical matches, uninspected leads and failed providers. It measures +top-one/top-three relevance, false clean-build outcomes, blocked winners and +ranking latency. It performs no network requests or candidate execution. +It is a regression gate, **not measured real-world recall or a human-reviewed +accuracy benchmark**. Provider-query behavior and evidence failures have separate +fixture-backed unit and integration tests. -Popularity cannot outweigh a missing license, archived status, failed sandbox test, poor feature match, or OSV evidence. OSV findings are conservatively blocking until severity and remediation are reviewed. Automatic reuse uses a conservative permissive-license allowlist; reciprocal, custom, or ambiguous licenses are blocked pending explicit compatibility review. - -## Recommendation contract - -Each candidate receives one disposition: - -- `use-as-library` — package candidate clears score/feature/security thresholds -- `fork` — GitHub repository has strong direct fit, evidence, and reusable licensing -- `selective-reuse` — licensed candidate has useful modules but needs meaningful adaptation -- `reject` — candidate is blocked by license, archive, OSV, or sandbox-test evidence -- `build-clean` — candidate does not clear the reuse threshold - -Each disposition also carries `ready`, `provisional`, `blocked`, or `not-selected` status. The run-level result chooses the highest-ranked candidate that clears reuse gates. If none clears them, it says `build-clean` and records the top score and disposition. - -## Repository inspection - -`--inspect-top N` is bounded to five candidates. A local candidate is statically inspected in place under the same file/read/size caps. For each public GitHub candidate Brief2Ship: - -1. accepts only a canonical public HTTPS GitHub repository URL; -2. fetches current repository, contributor, real-watcher, homepage/demo, true issue-only count, and community-profile security evidence; -3. blocks archived repositories; -4. blocks repositories whose reported size is unknown or above 250,000 KB; -5. shallow-clones one branch with no tags, no submodules, hooks disabled, and a 1 MB blob filter; -6. ignores symlinks and generated/vendor trees; -7. caps static traversal at 20,000 files and individual evidence reads at 2 MB; -8. checks package/build manifests, dependency declarations, license text, test files, CI workflows, docs, examples, languages, source footprint, and commit hash; -9. reruns deterministic scoring with the new evidence. - -Inspection does not execute repository code. +CI additionally runs pinned Ruff/Pyright checks, benchmark validation, installed +wheel round-trip smoke and full release validation from the extracted sdist. +The Linux/Windows Python matrix remains 3.11, 3.12 and 3.13. A local pass does not +claim GitHub Actions passed before this branch is pushed. ## Sandboxed tests -Untrusted code execution is off by default. It needs all of: - -```bash -brief2ship discover "..." \ - --inspect-top 3 \ - --test-top 1 \ - --allow-untrusted-tests \ - --output discovery/ -``` - -The current execution sandbox requires Linux, Bubblewrap, `prlimit`, and `timeout`. It: - -- disables network access and unshares network, PID, IPC, UTS, cgroup, and user namespaces; -- clears inherited environment variables and secrets; -- does not mount the host home or Windows drives; -- mounts only system runtime directories read-only; -- mounts the disposable candidate copy, empty `/tmp`, and empty `/home` read-only, then remounts the sandbox root read-only; -- caps additional processes, per-process address space (512 MiB), CPU time, per-file output size, and wall time; -- uses offline package-manager settings; -- captures bounded output tails and exit status. - -No dependencies are downloaded or installed during a test. The filesystem is intentionally read-only, so tests that require build artifacts, caches, or temporary files fail conservatively. Tests may also fail because a dependency is absent; the receipt distinguishes failure from evidence absence. If the sandbox is unavailable, testing is `blocked`. Brief2Ship never falls back to unsandboxed execution. - -## Limits and caveats - -- Public APIs can rate-limit, change fields, or temporarily fail. Other sources continue and every failure is recorded. Calls are capped by a 30-second per-request maximum, a 20-result per-source maximum, and one shared `--total-timeout` network budget (180 seconds by default, 600 maximum). -- Local search is bounded and read-only but can expose absolute local paths in its receipt; do not publish those receipts without review. -- GitHub search without a token is limited to its public unauthenticated quota. -- PyPI discovery initially downloads a large official name index because no challenge-free search API exists. -- Dependency counts are declared direct runtime evidence; optional extras and detected development-only dependencies are excluded where provider metadata distinguishes them. Counts are not complete transitive dependency graphs. -- OSV reports known database entries, not proof that a package is vulnerability-free. -- Static test/CI/docs signals do not prove quality; sandbox pass/fail is stronger but still bounded evidence. -- License identifiers and detected license text require human review before commercial reuse. -- Private results are filtered. Deprecated npm packages, yanked crates, gated artifacts, and disabled artifacts remain blocked evidence and cannot receive an automatic reuse recommendation. -- Hugging Face model/dataset quality cannot be inferred from downloads alone. -- This is code-solution triage, not a substitute for final architecture, legal, security, or performance review. - -## Free and bounded promise - -The core uses Python's standard library and public endpoints. No paid API is required. No browser automation, client-challenge bypass, CAPTCHA solving, proxy rotation, fingerprint evasion, credential harvesting, or authenticated-session scraping is used. +Tests remain opt-in using **both** `--test-top N` and `--allow-untrusted-tests`. +They require the supported Linux Bubblewrap sandbox, no network, cleared +environment, read-only candidate/work/temp/home filesystems and bounded +CPU/process/memory/output/wall time. Missing controls block execution; there is +no unsafe fallback. Static inspection and passing metadata gates never imply +candidate tests ran. Native Windows onboarding does not claim Bubblewrap support. + +## Sources and candidate dispositions + +- **Local workspaces:** repeatable `--local PATH` roots, bounded read-only breadth-first traversal, + no symlinked Git metadata, and in-place static inspection. +- **GitHub:** public repositories with license/activity, stars, forks, watchers, + contributor and issue evidence where available. **Private results are filtered** + even when an optional `GH_TOKEN` or `GITHUB_TOKEN` is present. +- **PyPI:** bounded name-index discovery and project JSON, not full-text semantic package search. +- **npm:** package search and version-specific detail hydration. +- **crates.io:** relevant Rust crate metadata and dependency evidence. +- **Hugging Face:** model, dataset and Space metadata; gated/disabled artifacts remain blocked. + +Candidate dispositions remain `use-as-library`, `fork`, `selective-reuse`, `reject` +and `build-clean`. A candidate-level recommendation is not itself a run-level +decision. Read `recommendation_status`, evidence coverage and required checks; +the overall run can remain inconclusive even with a nonempty shortlist. diff --git a/docs/plans/reliable-reuse-decisions.md b/docs/plans/reliable-reuse-decisions.md new file mode 100644 index 0000000..8d8269f --- /dev/null +++ b/docs/plans/reliable-reuse-decisions.md @@ -0,0 +1,45 @@ +# Reliable reuse decisions implementation plan + +## License evidence refinement after adversarial review + +Canonical MIT body recognition is separate from reuse authority. Free-form +copyright holder fields can contain arbitrary restrictions; a regex cannot +distinguish those reliably from names. Such text keeps `normalized_license=null`, +records `license_body_match=MIT`, and requires review. Exact identifiers and +complete MIT bodies with only fixed title prefixes remain supported. This +conservative tradeoff also affects ordinary copyright-bearing MIT files. + +## Target and constraints + +Improve Brief2Ship's existing dependency-free Python CLI for human and agent operators: reliable reuse decisions from bounded local/public discovery, honest unknown evidence, complete inspection receipts, deterministic relevance evaluation, concise output, and verified installation. No new registries, mandatory models, candidate execution, public pushes, tags, or publication. + +## Base selection + +- Baseline: `a708480fadcdbd832f7ec1428fe6b76378be5654` (`v0.6.2`). +- Branch: `fix/reliable-reuse-decisions`. +- Fresh installed-CLI preflight: `/tmp/brief2ship-preflight-implementation-dl9iyhrt/discovery.json`. +- Choice: canonical `five0nit/brief2ship`; disposition `selective-reuse`. +- Inspected local candidate: MIT, 74.50/100, `selective-reuse`. +- Rejected alternatives: `abs-repository-core` 56.50, `abs-nosql-repository-core` 54.50, `active-protocol-discovery` 54.50: database abstractions/protocol discovery, not a base for this repository-discovery CLI. No dependency addition needed. + +## Acceptance contracts + +1. Missing/malformed package detail evidence remains unknown; provider-confirmed empty dependency collections alone count as zero. Preserve source warnings. Canonical full MIT license normalization preserves raw observed text and rejects modified/truncated or additional restrictive text. +2. Query planning is deterministic, bounded to at most three variants, retains core task/domain terms, and records removed constraint terms. No unbounded retries, external model, paid service, or candidate execution. Exact names remain searchable. +3. Raw score remains explainable; add confidence-adjusted decision score and relevance eligibility so unrelated healthy projects cannot crowd out relevant leads. Keep blockers enforced. Unknown requested constraints must be visible, not assumed met. +4. Final build/reuse decision must not depend on output `limit`. Failed/incomplete sources or insufficient static inspection produce inconclusive status, not an automatic clean-build instruction. No-candidates remains inconclusive unless an operator separately justifies greenfield work. Statically inspected reusable candidates remain provisional when runtime/OSV checks are outstanding; never label those ready. +5. Full JSON and candidate artifacts retain every evaluated candidate and attempted inspection. Markdown shortlist honors `limit` but retains all candidate evidence. Record selected candidate identity and why each inspection slot was spent. +6. Optional concise summary contains decision/status, selected/top candidate, raw/decision score and coverage, blockers, source health and all receipt paths. Preserve default path-only stdout. Inconclusive/failed discovery returns nonzero while writing receipts. +7. Add deterministic fixture-backed task-quality benchmark and regression tests for domain false positives, long requests, unknown evidence, source outages, no matches, license text, local/public dedup and display-limit invariance. Fixture outcomes are synthetic regression evidence, not human-reviewed adoption/accuracy claims. +8. Automate critical Ruff, Pyright, full source/extracted-sdist tests, wheel install and installed round-trip gates in CI. Verify local clean package install, existing network-safety controls and live discovery without running candidates. +9. Update docs/CHANGELOG and a local verification receipt. Keep PyPI pending until separately authorized authenticated publication; no remote changes. Commit only verified files from this clean branch. + +## Ownership + +- Provider lane: `discovery_providers.py`, new `discovery_query.py`, provider/query tests. No shared model edits. +- Scoring lane: `discovery_scoring.py`, new `discovery_licenses.py`, scoring/license tests. No shared model edits. +- Parent integrator: models, orchestration, rendering, CLI, integration/CLI tests, benchmark, documentation, CI, package verification and final commit. + +## Review and verification + +Each implementation lane captures failing regression then passing focused tests. Parent performs spec verification before independent quality review. Final shared-runtime changes invalidate prior gates; rerun full source, benchmark, package and installed CLI checks before commit. Preserve receipts outside build payloads when they contain artifact hashes. diff --git a/docs/releases/v0.6.2-discoverability-receipt.md b/docs/releases/v0.6.2-discoverability-receipt.md index bb5f95f..f160794 100644 --- a/docs/releases/v0.6.2-discoverability-receipt.md +++ b/docs/releases/v0.6.2-discoverability-receipt.md @@ -1,10 +1,12 @@ # Brief2Ship v0.6.2 Discoverability Release Receipt -**Status:** PRE-TAG RELEASE CANDIDATE — VERIFIED -**Release candidate status:** locally and remotely verified; tag/release pending +**Status:** GITHUB RELEASE PUBLISHED — PYPI PUBLICATION STILL PENDING +**Release candidate status:** historical candidate gates below were completed before publication +**Published GitHub release:** +**Published at:** 2026-09-04T23:00:25Z (read back from GitHub release API) **Release date:** 2026-09-05 AEST **Repository:** `five0nit/brief2ship` -**Planned tag:** `v0.6.2` +**Published tag:** `v0.6.2` **Verified package-source commit:** `b775a7d2325b3eeead249c14a2232ab441ecfce0` **Pull request:** **Exact package-source CI:** diff --git a/docs/releases/v0.7.0-local-verification.md b/docs/releases/v0.7.0-local-verification.md new file mode 100644 index 0000000..839085b --- /dev/null +++ b/docs/releases/v0.7.0-local-verification.md @@ -0,0 +1,103 @@ +# v0.7.0 reliable reuse decisions — local verification + +**Historical status at verification: VERIFIED LOCAL CANDIDATE — NOT PUSHED, TAGGED OR PUBLISHED** + +This is the pre-publication snapshot, not the current GitHub release status. +The published release may contain later release-documentation changes and rebuilt +artifacts; its attached checksums identify those distributions. + +Baseline: `a708480fadcdbd832f7ec1428fe6b76378be5654` (v0.6.2). +Branch: `fix/reliable-reuse-decisions`. +Verification date: 2026-09-05 UTC. + +## Scope and preflight + +User authorized implementation after the read-only audit. Existing dependency-free +Python core retained; no new runtime dependencies, default candidate execution, +dashboards, providers or mandatory model calls added. + +Fresh preflight ran before implementation/dependency installation: +`/tmp/brief2ship-preflight-implementation-dl9iyhrt/discovery.json`. +The inspected canonical Brief2Ship repository was selected for selective reuse. +Full preflight and review evidence is copied into the durable evidence directory below. + +## Changes + +- Unknown dependency data no longer becomes observed zero; package and repository + metadata remain separate evidence scopes. +- Failed/partial sources, malformed nested responses, truncated local scans and + insufficient inspection produce inconclusive decisions, never automatic clean builds. +- Decisions use the complete evaluated set. Display limits cannot alter the choice; + full receipts retain every evaluated candidate and inspection attempt. +- Bounded deterministic intent planning, GitHub rank fusion, core registry queries, + relevance gating, confidence-adjusted ranking and explicit constraint checks. +- v2 receipts, concise `--summary`, documented exit `5` for inconclusive runs. +- Native PowerShell quickstart, synthetic benchmark, pinned lint/type gates, + clean console-entrypoint smoke, extracted-sdist CI checks and release-state cleanup. + +## Verification results + +| Gate | Actual result | +|---|---| +| Python 3.11.15 Linux | 191 tests, OK | +| Python 3.12.3 Linux | 191 tests, OK | +| Python 3.13.14 Linux | 191 tests, OK | +| Native Windows / Python 3.13.0 | 191 tests discovered; OK with 4 platform skips | +| Synthetic quality cases | 24/24; zero false clean-build cases in this fixture set | +| Pyright 1.1.411 | 0 errors, 0 warnings | +| Ruff 0.16.6 critical checks | All checks passed | +| Release/document/compile gates | Passed | +| Wheel + source distribution | Built; Twine checks passed | +| Extracted source distribution | Full release validation passed | +| Clean installed Linux wheel | Actual console entrypoint and receipt round-trip passed | +| Native Windows source install | Doctor, actual console entrypoint and receipt round-trip passed | +| Independent quality recheck | APPROVED; previous three findings resolved | +| Artifact/source comparison | All 22 packaged Python files and 99 corresponding sdist files matched current source bytes | + +Synthetic cases are agent-authored regression controls, not a human-reviewed +accuracy benchmark. Optional extractor dependencies were not part of these local +gates. Hosted GitHub Actions were not run because this branch was not pushed. + +The local self-discovery smoke intentionally returns `inconclusive` / exit `5`: +Brief2Ship's copyright-bearing MIT file is recognized but requires notice review. +Tests assert that distinction rather than forcing a favorable reuse result. + +## Final installed-artifact live probe + +Query: `local robots-aware web scraper`. +All five public sources returned `ok`; 15 candidates evaluated, 5 displayed, +2 inspection attempts and both inspection records retained. Elapsed: 22.05 seconds. +Core query: `web scraper` — not the previous truncated `local robots aware`. +Result: `ts-web-scraper`, `use-as-library`, **provisional**. +Remaining checks explicitly include package-specific evidence, authorized sandbox +tests, and the requested `local` and `robots-aware` constraints. Candidate code was +not executed. This is a live plumbing/receipt check, not approval to adopt that package. + +## Important conservative license boundary + +Exact supported identifiers and complete MIT grant text with fixed titles may be +normalized. A complete MIT body with free-form copyright/surrounding prose gets +`license_body_match=MIT`, `normalized_license=null`, `license_review_required=true`. +No regex can reliably distinguish arbitrary holder names from arbitrary conditions. +Ordinary copyright-bearing MIT files therefore require review too. Diagnostic body +recognition must never be treated as license authority. + +## Artifacts and durable receipts + +Maintainer-local evidence directory, relative to the local data root: +`brief2ship/verification/v0.7.0-20260905T020126Z` + +- Wheel: `brief2ship-0.7.0-py3-none-any.whl` + SHA-256: `860a392e50370f31cfdb128ebe2dd16cc92ff609e9a002a5aaf86156df608851` +- Source: `brief2ship-0.7.0.tar.gz` + SHA-256: `3e5e77da512d89a75dc2382848a0fba5001220d838a3b5f4e399f64a901183b8` +- `manifest.json`: artifact and evidence file hashes. +- `python-matrix-final.json`, `benchmark-final.json`, `release-final.log`, + `sdist-final.log`, `windows/`, `live-final/`, `quality-recheck.md`. + +This evidence-only release receipt is excluded from source archives by the existing +MANIFEST rules. It does not change the verified software payload. + +At this historical checkpoint, the shared installed CLI remained v0.6.2. v0.7.0 +was exercised from source and isolated installed artifacts; no shared-tool rollout +or public release had yet been performed. diff --git a/docs/windows-powershell.md b/docs/windows-powershell.md new file mode 100644 index 0000000..aff4551 --- /dev/null +++ b/docs/windows-powershell.md @@ -0,0 +1,84 @@ +# Native Windows PowerShell quickstart + +Use PowerShell with Python 3.11, 3.12 or 3.13 installed. Start in a fresh +Brief2Ship v0.7.0 source checkout containing `pyproject.toml`: + +```powershell +git clone --branch v0.7.0 --depth 1 https://github.com/five0nit/brief2ship.git +if ($LASTEXITCODE -ne 0) { throw "Source checkout failed" } +Set-Location brief2ship +``` + +This GitHub release includes v2 decisions. PyPI publication remains separate. + +## Isolated source install + +The Python launcher selects a native interpreter. Substitute an installed +supported version for `-3.13` when needed. No virtual-environment activation or +PowerShell execution-policy change is needed. + +```powershell +py -3.13 -m venv .venv +if ($LASTEXITCODE -ne 0) { throw "venv creation failed" } + +& .\.venv\Scripts\python.exe -m pip install . +if ($LASTEXITCODE -ne 0) { throw "Brief2Ship installation failed" } + +& .\.venv\Scripts\brief2ship.exe doctor +if ($LASTEXITCODE -ne 0) { throw "doctor failed" } + +& .\.venv\Scripts\brief2ship.exe --version +if ($LASTEXITCODE -ne 0) { throw "version check failed" } + +& .\.venv\Scripts\python.exe scripts\validate-release.py +if ($LASTEXITCODE -ne 0) { throw "source release validation failed" } +``` + +If the `py` launcher is absent, use the full path to an installed Python 3.11–3.13 +executable for the first command. `python.exe` must be a real interpreter, not the +Microsoft Store alias. The remaining commands explicitly target `.venv`. + +## Local-only discovery round trip + +```powershell +$output = Join-Path $env:TEMP ("brief2ship-discovery-" + [guid]::NewGuid().ToString("N")) +& .\.venv\Scripts\brief2ship.exe discover "brief2ship" ` + --local (Get-Location).Path --sources local ` + --per-source 3 --limit 3 --inspect-top 1 --summary --output $output +$discoveryExit = $LASTEXITCODE +if ($discoveryExit -notin @(0, 5)) { throw "Discovery failed: $discoveryExit" } + +$receipt = Get-Content -LiteralPath (Join-Path $output "discovery.json") -Raw | ConvertFrom-Json +$receipt | Select-Object schema_version, discovery_status, decision_status, overall_recommendation +$receipt.evaluated_candidates | Select-Object name, recommendation, recommendation_status +``` + +The canonical checkout's free-form copyright notice produces an expected +`inconclusive` result with exit `5` and an explicit license-review check. Its MIT +body is recognized, but surrounding prose is not automatically approved. + +Exit 5 is a successful receipt write but an **inconclusive decision**. Read its +`incomplete_reasons`; never turn it into permission for a clean build. +Exit 0 can still be a provisional reuse lead with required checks outstanding. +Use a fresh output directory for every run. + +## Installed-package smoke + +```powershell +& .\.venv\Scripts\python.exe scripts\smoke-installed.py +if ($LASTEXITCODE -ne 0) { throw "Installed-package smoke failed" } +``` + +This verifies the import resolves to the installed distribution, exercises the +installed CLI and checks a real local discovery receipt. Source validation and +installed-package validation are separate checks. + +## Boundaries + +- This route uses native Windows Python and `.venv\Scripts\*.exe` entrypoints; + it does not require or claim WSL. +- Local static discovery does not execute candidate code or require network. +- These commands do not enable or claim native Windows Bubblewrap candidate tests. +- The optional Trafilatura adapter is not installed or validated by this core-only + quickstart. Optional adapter CI is a separate gate. +- See [decision/schema migration](code-discovery.md) before updating automation. diff --git a/pyproject.toml b/pyproject.toml index 1ad61ba..e163bbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "brief2ship" -version = "0.6.2" +version = "0.7.0" description = "Repo-first discovery for AI coding agents: search and score existing code before building, then ship with proof." readme = "README.md" requires-python = ">=3.11" diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..e82e4ae --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,5 @@ +{ + "include": ["src/brief2ship", "scripts"], + "pythonVersion": "3.11", + "typeCheckingMode": "basic" +} diff --git a/scripts/benchmark-discovery.py b/scripts/benchmark-discovery.py new file mode 100644 index 0000000..3558de4 --- /dev/null +++ b/scripts/benchmark-discovery.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Deterministic synthetic task-quality regression gate; no provider requests.""" +from __future__ import annotations + +import argparse +import json +import statistics +import sys +import time +from datetime import datetime +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from brief2ship.discovery_decision import decide +from brief2ship.discovery_models import Candidate, InspectionResult, SourceReceipt +from brief2ship.discovery_scoring import rank_candidates +from brief2ship.render import atomic_write + + +def run_benchmark() -> dict: + fixtures = json.loads((ROOT / "tests/fixtures/discovery-quality-v1.json").read_text(encoding="utf-8")) + now = datetime.fromisoformat(fixtures["clock"]) + rows = [] + for case in fixtures["cases"]: + target = Candidate( + source="github", name=case["expected"], url="https://github.com/" + case["expected"], + repository_url="https://github.com/" + case["expected"], description=case["description"], + license="MIT", updated_at=fixtures["clock"], dependency_count=2, language="Python", + ) + unrelated = Candidate( + source="github", name="fixture/aaa-unrelated", url="https://github.com/fixture/aaa-unrelated", + description="Highly popular general purpose utility", repository_url="https://github.com/fixture/aaa-unrelated", + license="MIT", updated_at=fixtures["clock"], stars=1_000_000, dependency_count=0, + security_policy=True, vulnerabilities_checked=True, test_signals=["tests"], + reuse_signals=["docs", "examples"], language="Python", + ) + archived = Candidate( + source="github", name="fixture/archived", url="https://github.com/fixture/archived", + description=case["description"], license="MIT", archived=True, stars=1_000_000, + ) + unlicensed = Candidate( + source="github", name="fixture/unlicensed", url="https://github.com/fixture/unlicensed", + description=case["description"], stars=1_000_000, + ) + pool = [unrelated, archived, unlicensed, target] + start = time.perf_counter() + ranked = rank_candidates(case["query"], pool, now=now) + elapsed_ms = (time.perf_counter() - start) * 1000 + before_inspection = decide(ranked, [SourceReceipt("github", "ok", 4, returned=4)]) + outage = decide(ranked, [SourceReceipt("github", "failed", 4, error="synthetic outage")]) + target.inspection = InspectionResult(repository_url=target.repository_url or "", status="inspected", commit="fixture") + ranked_after = rank_candidates(case["query"], pool, now=now) + after_inspection = decide(ranked_after, [SourceReceipt("github", "ok", 4, returned=4)]) + names = [item.name for item in ranked] + checks = { + "top1": names[0] == case["expected"], + "top3": case["expected"] in names[:3], + "no_false_clean_build": before_inspection.recommendation == "inconclusive" and outage.recommendation == "inconclusive", + "inspected_reuse": after_inspection.selected_id == target.canonical_id and after_inspection.recommendation in {"selective-reuse", "fork", "use-as-library"}, + "no_blocked_top": not ranked[0].hard_blockers, + } + rows.append({"id": case["id"], "query": case["query"], "ranked": names, + "checks": checks, "passed": all(checks.values()), "ranking_ms": round(elapsed_ms, 3)}) + times = sorted(row["ranking_ms"] for row in rows) + count = len(rows) + return { + "schema": "brief2ship-quality-report-v1", "scope": fixtures["provenance"], + "case_count": count, "passed_count": sum(row["passed"] for row in rows), + "top1_hits": sum(row["checks"]["top1"] for row in rows), + "top3_hits": sum(row["checks"]["top3"] for row in rows), + "false_clean_build_cases": sum(not row["checks"]["no_false_clean_build"] for row in rows), + "blocked_top_cases": sum(not row["checks"]["no_blocked_top"] for row in rows), + "ranking_median_ms": round(statistics.median(times), 3), + "ranking_p95_ms": times[min(count - 1, max(0, (95 * count + 99) // 100 - 1))], + "network_requests": 0, "candidate_executions": 0, + "passed": bool(rows) and all(row["passed"] for row in rows), "cases": rows, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + report = run_benchmark() + if args.output: + atomic_write(args.output, json.dumps(report, indent=2, sort_keys=True) + "\n") + print(json.dumps({key: value for key, value in report.items() if key != "cases"}, indent=2, sort_keys=True)) + if not report["passed"]: + print(json.dumps({"failed_cases": [row for row in report["cases"] if not row["passed"]]}, indent=2)) + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check-sdist.py b/scripts/check-sdist.py new file mode 100644 index 0000000..ec15dcb --- /dev/null +++ b/scripts/check-sdist.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Run the full release contract from the built source archive.""" +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +import tarfile +import tempfile +from pathlib import Path + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("archive", type=Path) + args = parser.parse_args() + environment = os.environ.copy() + environment.pop("PYTHONPATH", None) + with tempfile.TemporaryDirectory(prefix="brief2ship-sdist-check-") as temporary: + destination = Path(temporary) + with tarfile.open(args.archive, "r:gz") as archive: + archive.extractall(destination, filter="data") + roots = list(destination.iterdir()) + if len(roots) != 1 or not roots[0].is_dir(): + raise RuntimeError("source archive must contain exactly one project root") + result = subprocess.run( + [sys.executable, "scripts/validate-release.py"], cwd=roots[0], env=environment, check=False, + ) + return result.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/smoke-installed.py b/scripts/smoke-installed.py new file mode 100644 index 0000000..fa8b308 --- /dev/null +++ b/scripts/smoke-installed.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Exercise the installed console entrypoint, not an editable source import.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import sysconfig +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def _installed_entrypoint() -> Path: + suffix = sysconfig.get_config_var("EXE") or "" + return Path(sysconfig.get_path("scripts")) / ("brief2ship" + suffix) + + +def main() -> int: + environment = os.environ.copy() + environment.pop("PYTHONPATH", None) + with tempfile.TemporaryDirectory(prefix="brief2ship-installed-smoke-") as temporary: + identity = subprocess.run( + [sys.executable, "-c", "import brief2ship,json; print(json.dumps({'version':brief2ship.__version__,'file':brief2ship.__file__}))"], + cwd=temporary, env=environment, text=True, capture_output=True, check=True, + ) + package = json.loads(identity.stdout) + if Path(package["file"]).resolve().is_relative_to(ROOT / "src"): + raise RuntimeError("installed smoke imported the source checkout instead of a distribution") + entrypoint = _installed_entrypoint() + version = subprocess.run([str(entrypoint), "--version"], cwd=temporary, env=environment, + text=True, capture_output=True, check=True) + if package["version"] not in version.stdout: + raise RuntimeError("console entrypoint and imported package versions disagree") + result = subprocess.run( + [str(entrypoint), "discover", "brief2ship", "--local", str(ROOT), + "--sources", "local", "--inspect-top", "1", "--summary", "--output", str(Path(temporary) / "receipt")], + cwd=temporary, env=environment, text=True, capture_output=True, + ) + summary = json.loads(result.stdout) + if result.returncode != 5 or summary["decision"] != "inconclusive": + raise RuntimeError(f"installed local round-trip skipped the copyright-review gate: {summary}") + receipt = json.loads(Path(summary["receipts"]["json"]).read_text(encoding="utf-8")) + if not receipt["evaluated_candidates"] or not receipt["inspection_decisions"]: + raise RuntimeError("installed round-trip omitted evaluation or inspection evidence") + own = next(c for c in receipt["evaluated_candidates"] if Path(c["local_path"]).resolve() == ROOT) + if own["license_body_match"] != "MIT" or not own["license_review_required"] or own["normalized_license"] is not None: + raise RuntimeError("installed package lost license recognition or failed to keep it separate from authority") + print(json.dumps({"package": package, "entrypoint": str(entrypoint), "decision": summary["decision"], + "decision_status": summary["decision_status"], "license_body_match": own["license_body_match"], + "schema": receipt["schema_version"], "passed": True}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate-release.py b/scripts/validate-release.py index 2feef92..fb3989a 100644 --- a/scripts/validate-release.py +++ b/scripts/validate-release.py @@ -34,6 +34,7 @@ def main() -> int: [sys.executable, "-m", "unittest", "discover", "-s", "tests", "-v"], env=environment, ) + run("task-quality regression benchmark", [sys.executable, "scripts/benchmark-discovery.py"], env=environment) print("Brief2Ship release validation passed") return 0 diff --git a/skills/brief2ship/SKILL.md b/skills/brief2ship/SKILL.md index f20c01d..275eb3d 100644 --- a/skills/brief2ship/SKILL.md +++ b/skills/brief2ship/SKILL.md @@ -1,7 +1,7 @@ --- name: brief2ship description: "Use when repo/package/local-workspace search, a new build, report/document, or public-web research is needed. Brief2Ship is the sole repo-search skill: discover and score bases, choose a disposition, apply maintainability/design/scraping gates, implement, verify, and retain proof." -version: 0.6.2 +version: 0.7.0 author: Hermes Agent license: MIT platforms: [linux, macos, windows] @@ -104,6 +104,15 @@ printf 'Brief2Ship preflight receipt: %s\n' "$PREFLIGHT_DIR" Keep limits bounded. Preserve the exact receipt path, source failures, warnings, and unknown evidence. +For CLI v0.7+, inspect `decision_status` and `discovery_status` before acting. +Exit `5` means receipts were written but the decision is `inconclusive`; it is +not an operational error and never permission to build clean. Repair failed +sources or refine a bounded search/inspection scope. Exit `0` may still mean +provisional reuse with required checks outstanding. `--summary` optionally emits +compact JSON; full v2 receipts retain `evaluated_candidates` and all inspection +allocations regardless of the display-only `--limit`. Rank leads using relevance +and confidence-adjusted `decision_score`, not raw popularity or total alone. + #### Discovery sources Use sources appropriate to the target: diff --git a/src/brief2ship/__init__.py b/src/brief2ship/__init__.py index 9d68dc1..3d6a54b 100644 --- a/src/brief2ship/__init__.py +++ b/src/brief2ship/__init__.py @@ -1,6 +1,6 @@ """Brief2Ship public package.""" -__version__ = "0.6.2" +__version__ = "0.7.0" from .discovery import discover from .discovery_models import Candidate, DiscoveryConfig, DiscoveryResult diff --git a/src/brief2ship/cli.py b/src/brief2ship/cli.py index 1018dc4..a9fe0db 100644 --- a/src/brief2ship/cli.py +++ b/src/brief2ship/cli.py @@ -14,7 +14,7 @@ from .crawl import crawl_site from .discovery import discover as discover_candidates from .discovery_models import DiscoveryConfig -from .discovery_render import write_discovery +from .discovery_render import render_discovery_summary, write_discovery from .errors import Brief2ShipError, OutputError, PolicyError from .models import ScrapeConfig from .render import atomic_write, render_result, write_crawl @@ -118,6 +118,7 @@ def build_parser() -> argparse.ArgumentParser: ) discover.add_argument("query") discover.add_argument("--output", type=Path, required=True) + discover.add_argument("--summary", action="store_true", help="print a concise JSON decision and source-health summary instead of only the receipt path") discover.add_argument( "--sources", type=_sources, @@ -231,8 +232,11 @@ def _run_discover(args: argparse.Namespace) -> int: ) result = discover_candidates(args.query, config, output_dir=args.output) receipt = write_discovery(result, args.output.expanduser().resolve()) - sys.stdout.write(f"{receipt}\n") - return 0 if result.candidates else 4 + if args.summary: + sys.stdout.write(render_discovery_summary(result, args.output.expanduser().resolve())) + else: + sys.stdout.write(f"{receipt}\n") + return 5 if result.decision_status == "inconclusive" or result.discovery_status != "complete" else 0 def main(argv: list[str] | None = None) -> int: diff --git a/src/brief2ship/discovery.py b/src/brief2ship/discovery.py index a02f0b7..54497c9 100644 --- a/src/brief2ship/discovery.py +++ b/src/brief2ship/discovery.py @@ -4,14 +4,17 @@ import os from copy import deepcopy +from dataclasses import asdict from datetime import datetime, timezone from pathlib import Path from .discovery_http import DiscoveryHttpClient +from .discovery_decision import decide from .discovery_inspection import RepositoryInspector from .discovery_local import search_local from .discovery_models import Candidate, DiscoveryConfig, DiscoveryResult, SourceReceipt from .discovery_providers import PROVIDERS, enrich_osv, search_pypi +from .discovery_query import plan_query from .discovery_scoring import rank_candidates from .errors import OutputError @@ -54,6 +57,19 @@ def _merge_candidate(existing: Candidate, incoming: Candidate) -> Candidate: primary, secondary = incoming, existing else: primary, secondary = existing, incoming + package_scope = primary.source in package_sources and secondary.source in {"github", "local"} + preserved = {} + if package_scope: + scoped_fields = ( + "description", "version", "license", "license_kind", "updated_at", "published_at", "dependency_count", + "topics", "test_signals", "portability_signals", "reuse_signals", "downloads", + "vulnerabilities_checked", "vulnerabilities", "vulnerability_evidence", + ) + preserved = {name: deepcopy(getattr(primary, name)) for name in scoped_fields} + primary.repository_evidence = { + name: deepcopy(getattr(secondary, name)) + for name in (*scoped_fields, "source", "url", "stars", "forks", "watchers", "contributors") + } primary.aliases = sorted( set(primary.aliases + secondary.aliases + [f"{secondary.source}:{secondary.name}", secondary.url]) ) @@ -117,6 +133,8 @@ def _merge_candidate(existing: Candidate, incoming: Candidate) -> Candidate: primary.test_signals = sorted(set(primary.test_signals + secondary.test_signals)) primary.portability_signals = sorted(set(primary.portability_signals + secondary.portability_signals)) primary.reuse_signals = sorted(set(primary.reuse_signals + secondary.reuse_signals)) + for name, value in preserved.items(): + setattr(primary, name, value) return primary @@ -170,7 +188,7 @@ def _inspection_priority(candidate: Candidate) -> tuple[float, float, str, str]: if candidate.score else 0.0 ) - total = candidate.score.total if candidate.score else 0.0 + total = candidate.score.decision_score if candidate.score else 0.0 return (-feature_match, -total, candidate.source, candidate.name.lower()) @@ -188,6 +206,7 @@ def discover( raise ValueError("query must contain between 2 and 300 non-whitespace characters") destination = prepare_output_directory(output_dir) started_at = _now() + query_plan = plan_query(normalized_query) http = client or DiscoveryHttpClient( timeout=config.timeout, total_timeout=config.total_timeout, @@ -219,6 +238,11 @@ def discover( found = [] receipt = SourceReceipt(source, "failed", config.per_source, error=f"invalid source response: {exc}") candidates.extend(found) + if not receipt.queries: + receipt.queries = [normalized_query] + for ordinal, item in enumerate(found, 1): + if item.source_rank is None: + item.source_rank = ordinal receipts.append(receipt) candidates = deduplicate_candidates(candidates) limitations = [ @@ -233,33 +257,22 @@ def discover( if warning and candidate.source in {"pypi", "npm", "crates"}: limitations.append(f"OSV {candidate.source}:{candidate.name}: {warning}") ranked = rank_candidates(normalized_query, candidates) + inspected: list[tuple[Candidate, str]] = [] if config.inspect_top: inspector = RepositoryInspector(http, destination / "worktrees") shortlist = sorted(ranked, key=_inspection_priority)[: config.inspect_top] for index, candidate in enumerate(shortlist): + feature = candidate.score.components.get("feature_match", 0) if candidate.score else 0 + reason = f"slot {index + 1}: feature fit {feature:.2f}/25, then confidence-adjusted score" candidate.inspection = inspector.inspect(candidate, run_tests=index < config.test_top) + inspected.append((candidate, reason)) ranked = rank_candidates(normalized_query, ranked) - ranked = ranked[: config.limit] - if ranked: - top = ranked[0] - reusable = next( - ( - candidate - for candidate in ranked - if candidate.recommendation in {"use-as-library", "fork", "selective-reuse"} - and candidate.recommendation_status in {"ready", "provisional"} - ), - None, - ) - if reusable: - overall = reusable.recommendation - reason = f"candidate {reusable.name} scored {reusable.score.total if reusable.score else 0:.2f}/100 with {reusable.recommendation_status} evidence" - else: - overall = "build-clean" - reason = f"no candidate cleared reuse gates; top score was {top.score.total if top.score else 0:.2f}/100 ({top.recommendation})" - else: - overall = "build-clean" - reason = "no candidates were returned by the selected sources" + decision = decide(ranked, receipts) + # A selected candidate must remain visible even when only one row is shown. + displayed = ranked[: config.limit] + if decision.selected_id and not any(item.canonical_id == decision.selected_id for item in displayed): + selected = next(item for item in ranked if item.canonical_id == decision.selected_id) + displayed = [selected, *displayed[: config.limit - 1]] config_receipt = { "sources": list(config.sources), "local_roots": [ @@ -280,9 +293,21 @@ def discover( started_at=started_at, completed_at=_now(), config=config_receipt, - candidates=ranked, + candidates=displayed, sources=receipts, - overall_recommendation=overall, - recommendation_reason=reason, + overall_recommendation=decision.recommendation, + recommendation_reason=decision.reason, limitations=sorted(set(limitations)), + evaluated_candidates=ranked, + selected_candidate_id=decision.selected_id, + decision_status=decision.status, + discovery_status=decision.discovery_status, + incomplete_reasons=decision.incomplete_reasons, + query_plan=asdict(query_plan), + inspection_decisions=[{ + "candidate_id": item.canonical_id or item.name, + "candidate": item.name, + "reason": reason, + "status": item.inspection.status if item.inspection else "not-run", + } for item, reason in inspected], ) diff --git a/src/brief2ship/discovery_decision.py b/src/brief2ship/discovery_decision.py new file mode 100644 index 0000000..06f879b --- /dev/null +++ b/src/brief2ship/discovery_decision.py @@ -0,0 +1,85 @@ +"""Run-level decisions: incomplete searches never authorize a clean build.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from .discovery_models import Candidate, SourceReceipt + +_REUSE = {"use-as-library", "fork", "selective-reuse"} + + +@dataclass +class DecisionOutcome: + recommendation: str = "inconclusive" + status: str = "inconclusive" + discovery_status: str = "incomplete" + reason: str = "discovery evidence is incomplete" + selected_id: str | None = None + incomplete_reasons: list[str] = field(default_factory=list) + + +def decide(candidates: list[Candidate], sources: list[SourceReceipt]) -> DecisionOutcome: + """Decide across all evaluated candidates, never the display-only shortlist. + + A static inspection can establish a provisional reuse lead, not a runtime + verification. Complete negative inspections can justify build-clean within + the bounded search. Empty/failed searches establish neither outcome. + """ + outcome = DecisionOutcome() + failed = [source for source in sources if source.status != "ok" or source.error] + if not sources or failed: + outcome.discovery_status = "partial" if any( + source.status in {"ok", "partial"} for source in sources + ) else "failed" + outcome.incomplete_reasons = [ + f"{source.source}: {source.error or source.status}" for source in failed + ] or ["no source receipts available"] + outcome.reason = "source collection incomplete; reuse/build decision deferred" + return outcome + + outcome.discovery_status = "complete" + if not candidates: + outcome.reason = "no candidates found; absence of results is not proof that reuse is unsuitable" + outcome.incomplete_reasons = ["broaden or revise the query before deciding to build clean"] + return outcome + + reusable = [candidate for candidate in candidates if ( + candidate.recommendation in _REUSE + and candidate.recommendation_status in {"ready", "provisional"} + and candidate.inspection is not None + and candidate.inspection.status == "inspected" + )] + if reusable: + selected = next((item for item in reusable if item.recommendation_status == "ready"), reusable[0]) + outcome.recommendation = selected.recommendation + outcome.status = "complete" if selected.recommendation_status == "ready" else "provisional" + outcome.selected_id = selected.canonical_id + outcome.reason = ( + f"candidate {selected.name} selected with {outcome.status} evidence; " + "static inspection completed" + ) + return outcome + + missing = [candidate for candidate in candidates if ( + candidate.inspection is None + or candidate.inspection.status != "inspected" + or not candidate.normalized_license + or not candidate.score + or (candidate.score.components.get("feature_match", 0) >= 8 and not ( + candidate.archived or candidate.deprecated or candidate.disabled or candidate.gated + or candidate.vulnerabilities or (candidate.inspection.test_receipt and + candidate.inspection.test_receipt.status in {"failed", "timeout", "oom", "signaled", "zero_tests"}) + )) + )] + if missing: + outcome.incomplete_reasons = [ + f"{item.name}: static inspection, license or supported negative evidence incomplete" for item in missing + ] + outcome.reason = "no inspected reusable candidate; remaining evidence is insufficient for build-clean" + return outcome + + outcome.recommendation = "build-clean" + outcome.status = "complete" + outcome.reason = "all evaluated candidates inspected; none cleared reuse gates within this bounded search" + return outcome diff --git a/src/brief2ship/discovery_http.py b/src/brief2ship/discovery_http.py index 2178e99..9118db9 100644 --- a/src/brief2ship/discovery_http.py +++ b/src/brief2ship/discovery_http.py @@ -44,7 +44,7 @@ def __init__( timeout: float = 20.0, total_timeout: float | None = None, github_token: str | None = None, - user_agent: str = "Brief2ShipBot/0.6.2 (+https://github.com/five0nit/brief2ship)", + user_agent: str = "Brief2ShipBot/0.7.0 (+https://github.com/five0nit/brief2ship)", ) -> None: self.timeout = timeout self._expires_at = time.monotonic() + total_timeout if total_timeout is not None else None diff --git a/src/brief2ship/discovery_inspection.py b/src/brief2ship/discovery_inspection.py index d938f85..1dc97d6 100644 --- a/src/brief2ship/discovery_inspection.py +++ b/src/brief2ship/discovery_inspection.py @@ -18,6 +18,7 @@ from urllib.parse import quote, urlencode, urlsplit from .discovery_http import DiscoveryHttpClient, DiscoverySourceError +from .discovery_licenses import read_license_evidence from .discovery_models import Candidate, InspectionResult, TestReceipt from .discovery_providers import canonical_repository_url @@ -172,22 +173,7 @@ def _bounded_tree_size(root: Path) -> int: def _detect_license(root: Path) -> str | None: - for name in ("LICENSE", "LICENSE.md", "LICENSE.txt", "COPYING", "COPYING.md"): - text = _safe_text(root / name).lower() - if not text: - continue - if "mit license" in text: - return "MIT" - if "apache license" in text and "version 2" in text: - return "Apache-2.0" - if "mozilla public license" in text and "2.0" in text: - return "MPL-2.0" - if "gnu general public license" in text: - return "GPL" - if "redistribution and use in source and binary forms" in text: - return "BSD" - return name - return None + return read_license_evidence(root, _safe_text) def _dependencies_from_manifest(path: Path) -> int | None: @@ -627,9 +613,13 @@ def _apply_inspection_evidence( candidate: Candidate, result: InspectionResult, ) -> None: - if result.license and not candidate.license: + package_scope = candidate.source in {"pypi", "npm", "crates", "huggingface"} + if result.license and not package_scope: + if candidate.license and candidate.license != result.license: + candidate.repository_evidence["prior_metadata_license"] = candidate.license candidate.license = result.license - if result.dependency_count is not None: + candidate.license_kind = "file" + if result.dependency_count is not None and not package_scope: candidate.dependency_count = result.dependency_count if result.test_files: candidate.test_signals.append("repository test files") diff --git a/src/brief2ship/discovery_licenses.py b/src/brief2ship/discovery_licenses.py new file mode 100644 index 0000000..452c722 --- /dev/null +++ b/src/brief2ship/discovery_licenses.py @@ -0,0 +1,120 @@ +"""Strict, dependency-free normalization for reusable license evidence.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Callable + +# These are exact metadata aliases for the existing permissive allowlist. The +# lookup is case-insensitive, but it never treats a license name as a substring +# of a larger expression or custom license. +_LICENSE_ALIASES = { + "0bsd": "0BSD", + "apache license 2.0": "Apache-2.0", + "apache-2.0": "Apache-2.0", + "bsd-2-clause": "BSD-2-Clause", + "bsd-3-clause": "BSD-3-Clause", + "isc": "ISC", + "mit": "MIT", + "mit license": "MIT", + "the unlicense": "Unlicense", + "unlicense": "Unlicense", +} +_PERMISSIVE_LICENSES = frozenset(_LICENSE_ALIASES.values()) + +_MIT_BODY = '''Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.''' +_MIT_BODY_START = "Permission is hereby granted, free of charge," +_MIT_TITLES = frozenset({"mit", "mit license", "the mit license", "the mit license (mit)"}) + + + +def _collapse_whitespace(value: str) -> str: + """Ignore wrapping and line-ending differences, but not wording changes.""" + + return " ".join(value.split()) + + +def _is_permitted_mit_prefix(prefix: str) -> bool: + """Exact titles only; free-form notices cannot confer decision authority.""" + + lines = [line.strip() for line in prefix.splitlines() if line.strip()] + return all(line.casefold() in _MIT_TITLES for line in lines) + + +def license_body_match(value: str | None) -> str | None: + """Recognize a complete MIT body, NEVER approval of surrounding text. + + Arbitrary holder names and arbitrary conditions are both natural language. + A name-shape regex or vocabulary denylist cannot distinguish them reliably. + This diagnostic match is deliberately separate from normalize_license. + """ + if not value: + return None + start = value.find(_MIT_BODY_START) + if start >= 0 and _collapse_whitespace(value[start:]) == _collapse_whitespace(_MIT_BODY): + return "MIT" + return None + + +def normalize_license(value: str | None, *, metadata: bool = True) -> str | None: + """Return a canonical permissive SPDX identifier for exact evidence. + + Full MIT text is recognized only when the complete canonical grant, + inclusion condition, warranty disclaimer, and liability disclaimer are + present with no appended terms. Only exact title prefixes and whitespace + variations are authoritative. Free-form copyright notices still receive a + license_body_match, but need review; never silently erase arbitrary prose. + Unknown, compound, truncated, and modified text remains unnormalized. + """ + + if value is None: + return None + stripped = value.strip().lstrip("\ufeff") + if not stripped: + return None + + alias = _LICENSE_ALIASES.get(_collapse_whitespace(stripped).casefold()) + if alias is not None and metadata: + return alias + + body_offset = stripped.find(_MIT_BODY_START) + if body_offset < 0: + return None + prefix = stripped[:body_offset] + body = stripped[body_offset:] + if not _is_permitted_mit_prefix(prefix): + return None + if _collapse_whitespace(body) != _collapse_whitespace(_MIT_BODY): + return None + return "MIT" + + +def is_permissive_license(normalized_license: str | None) -> bool: + """Return whether already-normalized evidence is on the reuse allowlist.""" + + return normalized_license in _PERMISSIVE_LICENSES + + +def read_license_evidence(root: Path, reader: Callable[[Path], str]) -> str | None: + """Retain bounded, symlink-safe file evidence without guessing SPDX.""" + for name in ("LICENSE", "LICENSE.md", "LICENSE.txt", "COPYING", "COPYING.md"): + text = reader(root / name) + if text: + return text + return None diff --git a/src/brief2ship/discovery_local.py b/src/brief2ship/discovery_local.py index 8e2af63..36568c0 100644 --- a/src/brief2ship/discovery_local.py +++ b/src/brief2ship/discovery_local.py @@ -12,6 +12,7 @@ from datetime import datetime, timezone from pathlib import Path +from .discovery_licenses import read_license_evidence from .discovery_models import Candidate, SourceReceipt from .discovery_providers import canonical_repository_url from .discovery_scoring import tokenize @@ -100,22 +101,7 @@ def _description(path: Path) -> str: def _license(path: Path) -> str | None: - for name in ("LICENSE", "LICENSE.md", "LICENSE.txt", "COPYING", "COPYING.md"): - text = _safe_text(path / name).lower() - if not text: - continue - if "mit license" in text: - return "MIT" - if "apache license" in text and "version 2" in text: - return "Apache-2.0" - if "mozilla public license" in text and "2.0" in text: - return "MPL-2.0" - if "gnu general public license" in text: - return "GPL" - if "redistribution and use in source and binary forms" in text: - return "BSD" - return None - return None + return read_license_evidence(path, _safe_text) def _language(markers: set[str], filenames: set[str]) -> str | None: @@ -223,6 +209,7 @@ def _candidate(path: Path, root: Path, query_tokens: set[str], filenames: set[st repository_url=repository_url, description=description, license=_license(path), + license_kind="file", updated_at=_updated_at(path, marker_names), language=language, topics=sorted(marker_names), @@ -312,7 +299,8 @@ def warn_once(key: str, message: str) -> None: ) break entries.append(entry) - except OSError: + except OSError as exc: + warn_once("directory-read", f"local directory read failed: {exc}") continue if stop_reason is not None: break @@ -336,7 +324,8 @@ def warn_once(key: str, message: str) -> None: and not name.startswith(".") ): child_directories.append(Path(entry.path)) - except OSError: + except OSError as exc: + warn_once("entry-read", f"local directory entry read failed: {exc}") continue if depth < _MAX_DEPTH: queue.extend((child, depth + 1) for child in child_directories) @@ -366,6 +355,9 @@ def warn_once(key: str, message: str) -> None: ) candidates = found[:limit] receipt.returned = len(candidates) - if not candidates: + if receipt.warnings: + receipt.status = "partial" + receipt.error = "; ".join(receipt.warnings) + elif not candidates: receipt.status = "empty" return candidates, receipt diff --git a/src/brief2ship/discovery_models.py b/src/brief2ship/discovery_models.py index 1a12938..8df9620 100644 --- a/src/brief2ship/discovery_models.py +++ b/src/brief2ship/discovery_models.py @@ -13,6 +13,7 @@ class ScoreBreakdown: evidence: dict[str, list[str]] coverage: float = 0.0 unknown_cost: float = 0.0 + decision_score: float = 0.0 @dataclass @@ -93,6 +94,13 @@ class Candidate: hard_blockers: list[str] = field(default_factory=list) required_checks: list[str] = field(default_factory=list) canonical_id: str | None = None + normalized_license: str | None = None + source_rank: int | None = None + constraint_checks: list[str] = field(default_factory=list) + repository_evidence: dict[str, Any] = field(default_factory=dict) + license_kind: str = "metadata" + license_body_match: str | None = None + license_review_required: bool = False def to_dict(self) -> dict[str, Any]: return asdict(self) @@ -108,6 +116,7 @@ class SourceReceipt: warnings: list[str] = field(default_factory=list) error: str | None = None rate_limit_remaining: int | None = None + queries: list[str] = field(default_factory=list) @dataclass(frozen=True) @@ -163,9 +172,16 @@ class DiscoveryResult: overall_recommendation: str recommendation_reason: str limitations: list[str] = field(default_factory=list) - schema_version: str = "brief2ship-discovery-v1" - scoring_contract: str = "brief2ship-score-v1" + schema_version: str = "brief2ship-discovery-v2" + scoring_contract: str = "brief2ship-score-v2" sandbox_policy: str = "brief2ship-bwrap-v2" + evaluated_candidates: list[Candidate] = field(default_factory=list) + selected_candidate_id: str | None = None + decision_status: str = "inconclusive" + discovery_status: str = "incomplete" + incomplete_reasons: list[str] = field(default_factory=list) + query_plan: dict[str, Any] = field(default_factory=dict) + inspection_decisions: list[dict[str, str]] = field(default_factory=list) def to_dict(self) -> dict[str, Any]: return asdict(self) diff --git a/src/brief2ship/discovery_providers.py b/src/brief2ship/discovery_providers.py index b67d5dd..72afe3b 100644 --- a/src/brief2ship/discovery_providers.py +++ b/src/brief2ship/discovery_providers.py @@ -16,37 +16,12 @@ from .discovery_http import DiscoveryHttpClient, DiscoverySourceError from .discovery_models import Candidate, SourceReceipt -from .discovery_scoring import tokenize +from .discovery_query import plan_query _GITHUB_API = "https://api.github.com" _PYPI_SIMPLE = "https://pypi.org/simple/" _OSV_API = "https://api.osv.dev/v1/query" -_GITHUB_FOCUS_STOPWORDS = { - "and", - "app", - "application", - "build", - "building", - "code", - "ecosystem", - "existing", - "first", - "for", - "greenfield", - "library", - "multi", - "new", - "of", - "or", - "package", - "reuse", - "solution", - "support", - "the", - "tool", - "tools", - "with", -} +_NAME_STOPWORDS = {"a", "an", "and", "for", "of", "or", "the", "to", "with"} def _rate_limit_remaining(headers: dict[str, str]) -> int | None: @@ -126,6 +101,12 @@ def _repository_from_mapping(mapping: Any) -> str | None: return None +def _record_partial(receipt: SourceReceipt, message: str) -> None: + receipt.status = "partial" + receipt.warnings.append(message) + receipt.error = f"{receipt.error}; {message}" if receipt.error else message + + def _github_candidate(item: dict[str, Any]) -> Candidate: license_data = item.get("license") or {} return Candidate( @@ -150,17 +131,15 @@ def _github_candidate(item: dict[str, Any]) -> Candidate: ) -def _github_focus_query(query: str) -> str: - """Reduce long prose to one bounded, deterministic GitHub fallback.""" - tokens: list[str] = [] - for value in re.findall(r"[A-Za-z0-9][A-Za-z0-9_.+#-]*", query.lower()): - for token in re.findall(r"[a-z0-9][a-z0-9_.+#]*", value.replace("-", " ")): - if len(token) < 2 or token in _GITHUB_FOCUS_STOPWORDS or token in tokens: - continue - tokens.append(token) - if len(tokens) < 2: - return query - return " ".join(tokens[:3]) +def _name_tokens(value: str) -> set[str]: + return { + token + for token in re.findall( + r"[a-z0-9]+", + value.lower().replace("_", "-").replace(".", "-"), + ) + if len(token) > 1 and token not in _NAME_STOPWORDS + } def search_github( @@ -168,58 +147,97 @@ def search_github( limit: int, client: DiscoveryHttpClient, ) -> tuple[list[Candidate], SourceReceipt]: - endpoint = f"{_GITHUB_API}/search/repositories?{urlencode({'q': query, 'per_page': limit, 'page': 1})}" - receipt = SourceReceipt("github", "ok", limit, endpoints=[endpoint]) - try: - data, payload = client.get_json( - endpoint, - headers={"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"}, + planned_queries = plan_query(query).variants[:3] + receipt = SourceReceipt("github", "ok", limit) + candidates_by_key: dict[str, Candidate] = {} + fusion_scores: dict[str, float] = {} + best_ranks: dict[str, int] = {} + first_seen: dict[str, int] = {} + remaining_values: list[int] = [] + failures: list[str] = [] + private_count = 0 + low_rate_limit = False + successful_queries = 0 + + for query_index, planned_query in enumerate(planned_queries): + endpoint = ( + f"{_GITHUB_API}/search/repositories?" + f"{urlencode({'q': planned_query, 'per_page': limit, 'page': 1})}" ) - if not isinstance(data, dict) or not isinstance(data.get("items"), list): - raise DiscoverySourceError("GitHub response had no repository items") - remaining_values = [_rate_limit_remaining(payload.headers)] - focused_query = _github_focus_query(query) - if not data["items"] and focused_query != query: - focused_endpoint = f"{_GITHUB_API}/search/repositories?{urlencode({'q': focused_query, 'per_page': limit, 'page': 1})}" - focused_data, focused_payload = client.get_json( - focused_endpoint, + receipt.queries.append(planned_query) + receipt.endpoints.append(endpoint) + try: + data, payload = client.get_json( + endpoint, headers={ "Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", }, ) - if not isinstance(focused_data, dict) or not isinstance( - focused_data.get("items"), - list, - ): - raise DiscoverySourceError( - "GitHub focused fallback had no repository items" - ) - data = focused_data - payload = focused_payload - receipt.endpoints.append(focused_endpoint) - receipt.warnings.append( - f"original query returned no results; used focused fallback: {focused_query}" - ) - remaining_values.append(_rate_limit_remaining(payload.headers)) - public_items = [ - item for item in data["items"] - if isinstance(item, dict) and not item.get("private") - ] - private_count = len(data["items"]) - len(public_items) - candidates = [_github_candidate(item) for item in public_items] - observed_remaining = [value for value in remaining_values if value is not None] - receipt.rate_limit_remaining = min(observed_remaining) if observed_remaining else None - receipt.returned = len(candidates) - if payload.headers.get("x-ratelimit-limit") == "10": - receipt.warnings.append("unauthenticated GitHub rate limit is low; set GH_TOKEN or GITHUB_TOKEN") - if private_count: - receipt.warnings.append(f"filtered {private_count} private GitHub result(s)") - return candidates, receipt - except DiscoverySourceError as exc: + except DiscoverySourceError as exc: + message = f"query {planned_query!r}: {exc}" + failures.append(message) + receipt.warnings.append(message) + continue + if not isinstance(data, dict) or not isinstance(data.get("items"), list): + message = f"query {planned_query!r}: response had no repository items" + failures.append(message) + receipt.warnings.append(message) + continue + + successful_queries += 1 + remaining = _rate_limit_remaining(payload.headers) + if remaining is not None: + remaining_values.append(remaining) + low_rate_limit = low_rate_limit or payload.headers.get("x-ratelimit-limit") == "10" + + public_rank = 0 + for item in data["items"]: + if not isinstance(item, dict): + continue + if item.get("private"): + private_count += 1 + continue + public_rank += 1 + candidate = _github_candidate(item) + identity = (candidate.repository_url or candidate.name).lower().rstrip("/") + if not identity: + continue + if identity not in candidates_by_key: + candidates_by_key[identity] = candidate + first_seen[identity] = query_index + best_ranks[identity] = public_rank + fusion_scores[identity] = 0.0 + fusion_scores[identity] += 1.0 / (60 + public_rank) + best_ranks[identity] = min(best_ranks[identity], public_rank) + + ordered_keys = sorted( + candidates_by_key, + key=lambda key: ( + -fusion_scores[key], + best_ranks[key], + first_seen[key], + candidates_by_key[key].name.lower(), + ), + )[:limit] + candidates = [candidates_by_key[key] for key in ordered_keys] + for source_rank, candidate in enumerate(candidates, start=1): + candidate.source_rank = source_rank + + receipt.returned = len(candidates) + receipt.rate_limit_remaining = min(remaining_values) if remaining_values else None + if successful_queries == 0: receipt.status = "failed" - receipt.error = str(exc) - return [], receipt + receipt.error = "; ".join(failures) or "all GitHub queries failed" + elif failures: + receipt.status = "partial" + if low_rate_limit: + receipt.warnings.append( + "unauthenticated GitHub rate limit is low; set GH_TOKEN or GITHUB_TOKEN" + ) + if private_count: + receipt.warnings.append(f"filtered {private_count} private GitHub result(s)") + return candidates, receipt def _load_pypi_names( @@ -270,8 +288,8 @@ def _load_pypi_names( def _name_fit(query: str, name: str) -> tuple[int, int, str]: - query_tokens = tokenize(query) - name_tokens = tokenize(name.replace("-", " ")) + query_tokens = _name_tokens(query) + name_tokens = _name_tokens(name.replace("-", " ")) overlap = len(query_tokens & name_tokens) all_match = int(bool(query_tokens) and query_tokens <= name_tokens) return all_match, overlap, name.lower() @@ -292,13 +310,11 @@ def _pypi_repository(info: dict[str, Any]) -> str | None: def _pypi_runtime_dependency_count(requires: object) -> int | None: - if not isinstance(requires, list): + if not isinstance(requires, list) or any( + not isinstance(value, str) for value in requires + ): return None - return sum( - 1 - for value in requires - if isinstance(value, str) and "extra ==" not in value.lower() - ) + return sum(1 for value in requires if "extra ==" not in value.lower()) def search_pypi( @@ -309,12 +325,19 @@ def search_pypi( cache_dir: Path, refresh: bool = False, ) -> tuple[list[Candidate], SourceReceipt]: - receipt = SourceReceipt("pypi", "ok", limit, endpoints=[_PYPI_SIMPLE]) + search_query = plan_query(query).core_query + receipt = SourceReceipt( + "pypi", + "ok", + limit, + endpoints=[_PYPI_SIMPLE], + queries=[search_query], + ) try: names = _load_pypi_names(client, cache_dir, refresh, receipt.warnings) matched_names: list[tuple[tuple[int, int, str], str]] = [] for name in names: - fit = _name_fit(query, name) + fit = _name_fit(search_query, name) if fit[1] > 0: matched_names.append((fit, name)) ranked_names = [ @@ -327,15 +350,24 @@ def search_pypi( candidates: list[Candidate] = [] for name in ranked_names[: max(limit * 3, limit)]: endpoint = f"https://pypi.org/pypi/{quote(name, safe='')}/json" + receipt.endpoints.append(endpoint) try: data, _ = client.get_json(endpoint, max_bytes=3_000_000) except DiscoverySourceError as exc: - receipt.warnings.append(f"{name}: {exc}") + _record_partial(receipt, f"{name}: {exc}") continue if not isinstance(data, dict) or not isinstance(data.get("info"), dict): + _record_partial(receipt, f"{name}: package detail had no info object") continue info = data["info"] - requires = info.get("requires_dist") or [] + requires = info.get("requires_dist") + if requires is not None and ( + not isinstance(requires, list) + or any(not isinstance(value, str) for value in requires) + ): + receipt.warnings.append( + f"{name}: requires_dist was malformed; dependency count is unknown" + ) releases = data.get("releases") or {} version = str(info.get("version") or "") or None upload_time = None @@ -347,26 +379,33 @@ def search_pypi( Candidate( source="pypi", name=str(info.get("name") or name), - url=str(info.get("package_url") or f"https://pypi.org/project/{name}/"), + url=str( + info.get("package_url") + or f"https://pypi.org/project/{name}/" + ), repository_url=_pypi_repository(info), description=str(info.get("summary") or ""), version=version, - license=str(info.get("license_expression") or info.get("license") or "") or None, + license=str( + info.get("license_expression") or info.get("license") or "" + ) + or None, updated_at=upload_time, published_at=upload_time, language="Python", - topics=[str(value) for value in info.get("keywords") or []] if isinstance(info.get("keywords"), list) else str(info.get("keywords") or "").split(), + topics=( + [str(value) for value in info.get("keywords") or []] + if isinstance(info.get("keywords"), list) + else str(info.get("keywords") or "").split() + ), dependency_count=_pypi_runtime_dependency_count(requires), reuse_signals=["package metadata", "installable package"], + source_rank=len(candidates) + 1, ) ) - receipt.endpoints.append(endpoint) if len(candidates) >= limit: break receipt.returned = len(candidates) - if not candidates: - receipt.status = "failed" - receipt.error = "PyPI name search produced no inspectable candidates" return candidates, receipt except DiscoverySourceError as exc: receipt.status = "failed" @@ -375,31 +414,48 @@ def search_pypi( def search_npm(query: str, limit: int, client: DiscoveryHttpClient) -> tuple[list[Candidate], SourceReceipt]: + query = plan_query(query).core_query endpoint = f"https://registry.npmjs.org/-/v1/search?{urlencode({'text': query, 'size': limit})}" - receipt = SourceReceipt("npm", "ok", limit, endpoints=[endpoint]) + receipt = SourceReceipt("npm", "ok", limit, endpoints=[endpoint], queries=[query]) try: data, _ = client.get_json(endpoint) objects = data.get("objects") if isinstance(data, dict) else None if not isinstance(objects, list): raise DiscoverySourceError("npm response had no objects") candidates: list[Candidate] = [] - for entry in objects: + for entry in objects[:limit]: package = entry.get("package") if isinstance(entry, dict) else None if not isinstance(package, dict): continue name = str(package.get("name") or "") detail_url = f"https://registry.npmjs.org/{quote(name, safe='')}" detail: dict[str, Any] = {} + hydrated = False + receipt.endpoints.append(detail_url) try: detail_data, _ = client.get_json(detail_url, max_bytes=8_000_000) if isinstance(detail_data, dict): detail = detail_data - receipt.endpoints.append(detail_url) + hydrated = True + else: + _record_partial(receipt, f"{name}: package detail was malformed") except DiscoverySourceError as exc: - receipt.warnings.append(f"{name}: {exc}") + _record_partial(receipt, f"{name}: {exc}") version = str(package.get("version") or "") or None - version_data = (detail.get("versions") or {}).get(version, {}) if version else {} - dependencies = version_data.get("dependencies") or {} + versions = detail.get("versions") + raw_version = versions.get(version) if isinstance(versions, dict) and version else None + version_known = hydrated and isinstance(raw_version, dict) + version_data = raw_version if isinstance(raw_version, dict) else {} + # A missing field in a known version means none; a missing version + # or malformed field means unknown. Never reward failed hydration. + dependencies = version_data.get("dependencies", {}) if version_known else None + dependency_count = None + if isinstance(dependencies, dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in dependencies.items()): + dependency_count = len(dependencies) + elif version_known: + _record_partial(receipt, f"{name}: dependencies malformed; count unknown") + else: + _record_partial(receipt, f"{name}: version {version or 'unknown'} metadata unavailable; dependency count unknown") dev_dependencies = version_data.get("devDependencies") or {} scripts = version_data.get("scripts") or {} deprecation_reason = str(version_data.get("deprecated") or detail.get("deprecated") or "") @@ -418,7 +474,7 @@ def search_npm(query: str, limit: int, client: DiscoveryHttpClient) -> tuple[lis updated_at=package.get("date"), downloads=int(((entry.get("downloads") or {}).get("monthly") or (entry.get("downloads") or {}).get("weekly") or 0)) if isinstance(entry.get("downloads") or {}, dict) else None, language="JavaScript", - dependency_count=len(dependencies) if isinstance(dependencies, dict) else None, + dependency_count=dependency_count, test_signals=( ["npm test script", f"development dependencies={len(dev_dependencies)}"] if isinstance(scripts, dict) and scripts.get("test") and isinstance(dev_dependencies, dict) @@ -428,6 +484,7 @@ def search_npm(query: str, limit: int, client: DiscoveryHttpClient) -> tuple[lis deprecation_reason=deprecation_reason or None, raw_relevance=float(score_data.get("final") or 0), reuse_signals=["package metadata", "installable package"], + source_rank=len(candidates) + 1, ) ) receipt.returned = len(candidates) @@ -439,15 +496,16 @@ def search_npm(query: str, limit: int, client: DiscoveryHttpClient) -> tuple[lis def search_crates(query: str, limit: int, client: DiscoveryHttpClient) -> tuple[list[Candidate], SourceReceipt]: + query = plan_query(query).core_query endpoint = f"https://crates.io/api/v1/crates?{urlencode({'q': query, 'sort': 'relevance', 'per_page': limit, 'page': 1})}" - receipt = SourceReceipt("crates", "ok", limit, endpoints=[endpoint]) + receipt = SourceReceipt("crates", "ok", limit, endpoints=[endpoint], queries=[query]) try: data, _ = client.get_json(endpoint) crates = data.get("crates") if isinstance(data, dict) else None if not isinstance(crates, list): raise DiscoverySourceError("crates.io response had no crates") candidates: list[Candidate] = [] - for item in crates: + for item in crates[:limit]: if not isinstance(item, dict): continue name = str(item.get("id") or item.get("name") or "") @@ -455,6 +513,7 @@ def search_crates(query: str, limit: int, client: DiscoveryHttpClient) -> tuple[ dependency_count = None if version: dependency_url = f"https://crates.io/api/v1/crates/{quote(name, safe='')}/{quote(version, safe='')}/dependencies" + receipt.endpoints.append(dependency_url) try: dependency_data, _ = client.get_json(dependency_url) dependencies = dependency_data.get("dependencies") if isinstance(dependency_data, dict) else None @@ -466,12 +525,13 @@ def search_crates(query: str, limit: int, client: DiscoveryHttpClient) -> tuple[ and dependency.get("kind") in {None, "normal"} and not dependency.get("optional") ) - if isinstance(dependencies, list) + if isinstance(dependencies, list) and all(isinstance(dep, dict) for dep in dependencies) else None ) - receipt.endpoints.append(dependency_url) + if dependency_count is None: + _record_partial(receipt, f"{name}: dependency response malformed; count unknown") except DiscoverySourceError as exc: - receipt.warnings.append(f"{name}: {exc}") + _record_partial(receipt, f"{name}: {exc}") candidates.append( Candidate( source="crates", @@ -510,8 +570,10 @@ def _hf_license(item: dict[str, Any]) -> str | None: def search_huggingface(query: str, limit: int, client: DiscoveryHttpClient) -> tuple[list[Candidate], SourceReceipt]: - receipt = SourceReceipt("huggingface", "ok", limit) + query = plan_query(query).core_query + receipt = SourceReceipt("huggingface", "ok", limit, queries=[query]) candidates_by_kind: list[list[Candidate]] = [] + successful_kinds = 0 per_kind = max(1, math.ceil(limit / 3)) for kind, plural in (("model", "models"), ("dataset", "datasets"), ("space", "spaces")): kind_candidates: list[Candidate] = [] @@ -520,7 +582,7 @@ def search_huggingface(query: str, limit: int, client: DiscoveryHttpClient) -> t try: data, payload = client.get_json(endpoint, max_bytes=8_000_000) except DiscoverySourceError as exc: - receipt.warnings.append(f"{kind}: {exc}") + _record_partial(receipt, f"{kind}: {exc}") candidates_by_kind.append(kind_candidates) continue remaining = _rate_limit_remaining(payload.headers) @@ -531,24 +593,40 @@ def search_huggingface(query: str, limit: int, client: DiscoveryHttpClient) -> t else min(receipt.rate_limit_remaining, remaining) ) if not isinstance(data, list): - receipt.warnings.append(f"{kind}: response was not a list") + _record_partial(receipt, f"{kind}: response was not a list") candidates_by_kind.append(kind_candidates) continue - for position, item in enumerate(data): + successful_kinds += 1 + for position, item in enumerate(data[:per_kind]): if not isinstance(item, dict): + _record_partial(receipt, f"{kind}: result was not an object") + continue + card = item.get("cardData") + tags_data = item.get("tags") + invalid = ( + not isinstance(item.get("id"), str) or not item.get("id") + or (card is not None and not isinstance(card, dict)) + or (tags_data is not None and ( + not isinstance(tags_data, list) or not all(isinstance(tag, str) for tag in tags_data) + )) + or any(item.get(field) is not None and type(item[field]) is not int for field in ("downloads", "likes")) + or (item.get("gated") is not None and not isinstance(item["gated"], (bool, str))) + ) + if invalid: + _record_partial(receipt, f"{kind}: malformed result fields at rank {position + 1}") continue if item.get("private"): receipt.warnings.append(f"{kind}: filtered private result") continue identifier = str(item.get("id") or "") - tags = [str(value) for value in item.get("tags") or []] + tags = [str(value) for value in tags_data or []] kind_candidates.append( Candidate( source="huggingface", name=identifier, url=f"https://huggingface.co/{'datasets/' if kind == 'dataset' else 'spaces/' if kind == 'space' else ''}{identifier}", repository_url=None, - description=str((item.get("cardData") or {}).get("summary") or item.get("pipeline_tag") or ""), + description=str((card if isinstance(card, dict) else {}).get("summary") or item.get("pipeline_tag") or ""), license=_hf_license(item), updated_at=item.get("lastModified") or item.get("last_modified"), downloads=int(item.get("downloads") or 0), @@ -570,9 +648,9 @@ def search_huggingface(query: str, limit: int, client: DiscoveryHttpClient) -> t if index < len(group) ][:limit] receipt.returned = len(candidates) - if not candidates and receipt.warnings: + if successful_kinds == 0: receipt.status = "failed" - receipt.error = "all Hugging Face endpoints failed" + receipt.error = f"all Hugging Face endpoints failed: {receipt.error or 'invalid responses'}" return candidates, receipt diff --git a/src/brief2ship/discovery_query.py b/src/brief2ship/discovery_query.py new file mode 100644 index 0000000..9a9a9a1 --- /dev/null +++ b/src/brief2ship/discovery_query.py @@ -0,0 +1,248 @@ +"""Deterministic, bounded query planning for discovery providers.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +_TOKEN_RE = re.compile( + r"@[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+" + r"|[A-Za-z0-9_.+#-]+/[A-Za-z0-9_.+#-]+" + r"|[A-Za-z0-9][A-Za-z0-9_.+#-]*" +) +_VERSION_RE = re.compile(r"\d+(?:\.\d+)*(?:\+)?$") + +_LEADING_INSTRUCTION_WORDS = { + "a", + "an", + "build", + "create", + "develop", + "discover", + "existing", + "find", + "help", + "i", + "implement", + "looking", + "make", + "me", + "need", + "new", + "please", + "search", + "the", + "use", + "want", + "we", +} +_ARTIFACT_WORDS = {"app", "application", "library", "package", "project", "solution", "tool"} +_TRAILING_CONNECTORS = {"and", "for", "on", "or", "using", "with"} +_FOCUS_STOPWORDS = { + "a", + "an", + "and", + "app", + "application", + "code", + "existing", + "first", + "for", + "library", + "new", + "of", + "or", + "package", + "project", + "reuse", + "solution", + "support", + "the", + "tool", + "using", + "with", +} +_CONSTRAINT_TERMS = { + "bounded", + "browser-based", + "cloud-free", + "cross-platform", + "dependency-free", + "deterministic", + "free", + "local", + "locally", + "mobile-first", + "no-cloud", + "no-dependency", + "offline", + "open-source", + "private-only", + "public-only", + "read-only", + "robots-aware", + "robots-compliant", + "self-contained", + "self-hosted", + "zero-dependency", +} +_RUNTIME_TERMS = { + ".net", + "android", + "c#", + "c++", + "deno", + "go", + "golang", + "ios", + "java", + "javascript", + "kotlin", + "linux", + "macos", + "node", + "node.js", + "php", + "python", + "ruby", + "rust", + "swift", + "typescript", + "windows", +} +_NEGATED_RESOURCES = { + "authentication", + "cloud", + "database", + "dependencies", + "dependency", + "docker", + "gpu", + "internet", + "network", +} +_RESOURCE_SUFFIXES = {"access", "service", "services"} + + +@dataclass(frozen=True) +class QueryPlan: + original: str + core_query: str + constraints: tuple[str, ...] + variants: tuple[str, ...] + + +def _tokens(value: str) -> list[str]: + return _TOKEN_RE.findall(value) + + +def _is_constraint_term(value: str) -> bool: + lowered = value.lower() + if lowered in _CONSTRAINT_TERMS or lowered in _RUNTIME_TERMS: + return True + parts = lowered.split("/") + return len(parts) > 1 and all( + part in _CONSTRAINT_TERMS or part in _RUNTIME_TERMS for part in parts + ) + + +def _constraint_ranges(tokens: list[str]) -> list[tuple[int, int]]: + """Return non-overlapping token ranges containing explicit constraints.""" + ranges: list[tuple[int, int]] = [] + index = 0 + while index < len(tokens): + lowered = tokens[index].lower() + next_lower = tokens[index + 1].lower() if index + 1 < len(tokens) else "" + + if lowered in _RUNTIME_TERMS and _VERSION_RE.fullmatch(next_lower): + ranges.append((index, index + 2)) + index += 2 + continue + if lowered in {"no", "without"} and next_lower in _NEGATED_RESOURCES: + end = index + 2 + if end < len(tokens) and tokens[end].lower() in _RESOURCE_SUFFIXES: + end += 1 + ranges.append((index, end)) + index = end + continue + if lowered in {"no", "without"} and next_lower == "paid": + end = index + 2 + if end < len(tokens) and tokens[end].lower() in {"api", "apis", "service", "services"}: + end += 1 + ranges.append((index, end)) + index = end + continue + if _is_constraint_term(tokens[index]): + ranges.append((index, index + 1)) + index += 1 + return ranges + + +def _core_tokens(tokens: list[str], ranges: list[tuple[int, int]]) -> list[str]: + constrained = { + index + for start, end in ranges + for index in range(start, end) + } + # Remove only connectors introducing a recognized constraint, not the + # connectors inside a task such as "PDF to Markdown" or "audio and video". + introducers = {"for", "with", "on", "in", "using", "that", "runs", "run", "and", "or", "must", "be"} + for start, _ in ranges: + previous = start - 1 + while previous >= 0 and tokens[previous].lower() in introducers: + constrained.add(previous) + previous -= 1 + values = [token for index, token in enumerate(tokens) if index not in constrained] + + while values and values[0].lower() in _LEADING_INSTRUCTION_WORDS: + values.pop(0) + if ( + len(values) >= 2 + and values[0].lower() in _ARTIFACT_WORDS + and values[1].lower() == "for" + ): + values = values[2:] + while values and values[-1].lower() in _TRAILING_CONNECTORS: + values.pop() + return values + + +def _focused_variant(core_query: str) -> str: + values = _tokens(core_query) + focused = [value for value in values if value.lower() not in _FOCUS_STOPWORDS] + if len(focused) < 2: + return core_query + return " ".join(focused) + + +def plan_query(query: str) -> QueryPlan: + """Split a request into core intent, recorded constraints, and at most three queries. + + Variants only remove observed terms or combine observed constraints with the core; + the planner does not invent synonyms or make semantic expansion claims. + """ + tokens = _tokens(query) + ranges = _constraint_ranges(tokens) + constraints = tuple(" ".join(tokens[start:end]) for start, end in ranges) + core_values = _core_tokens(tokens, ranges) + normalized = " ".join(tokens) + core_query = " ".join(core_values) or normalized or query + + variants: list[str] = [] + + def add(value: str) -> None: + if value and value not in variants and len(variants) < 3: + variants.append(value) + + add(core_query) + add(query) + add(_focused_variant(core_query)) + if constraints: + add(f"{constraints[-1]} {core_query}") + + return QueryPlan( + original=query, + core_query=core_query, + constraints=constraints, + variants=tuple(variants), + ) diff --git a/src/brief2ship/discovery_render.py b/src/brief2ship/discovery_render.py index 1b6a60e..9455590 100644 --- a/src/brief2ship/discovery_render.py +++ b/src/brief2ship/discovery_render.py @@ -29,6 +29,40 @@ def render_discovery_json(result: DiscoveryResult) -> str: return json.dumps(result.to_dict(), indent=2, sort_keys=True, ensure_ascii=False) + "\n" +def _all_candidates(result: DiscoveryResult) -> list[Candidate]: + return result.evaluated_candidates or result.candidates + + +def render_discovery_summary(result: DiscoveryResult, output_dir: Path) -> str: + """Compact JSON for operators; full receipts remain authoritative.""" + evaluated = _all_candidates(result) + selected = next((item for item in evaluated if item.canonical_id == result.selected_candidate_id), None) if result.selected_candidate_id else None + top = selected or (result.candidates[0] if result.candidates else None) + score = top.score if top else None + payload = { + "decision": result.overall_recommendation, + "decision_status": result.decision_status, + "discovery_status": result.discovery_status, + "selected_candidate": _safe(selected.name) if selected else None, + "top_candidate": _safe(top.name) if top else None, + "score": score.total if score else None, + "decision_score": score.decision_score if score else None, + "coverage": score.coverage if score else None, + "evaluated_count": len(evaluated), + "displayed_count": len(result.candidates), + "hard_blocker_count": sum(len(item.hard_blockers) for item in evaluated), + "required_checks": [_safe(value) for value in top.required_checks] if top else [], + "incomplete_reasons": [_safe(value) for value in result.incomplete_reasons], + "sources": [{ + "source": item.source, "status": item.status, "returned": item.returned, + "error": _safe(item.error) if item.error else None, + "warning_count": len(item.warnings), + } for item in result.sources], + "receipts": {kind: str(output_dir / f"discovery.{extension}") for kind, extension in (("markdown", "md"), ("json", "json"))}, + } + return json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + + def _score(candidate: Candidate, key: str) -> str: if not candidate.score: return "0.00" @@ -53,15 +87,22 @@ def render_discovery_markdown(result: DiscoveryResult) -> str: "## Decision", "", f"- Overall: {_code(result.overall_recommendation)}", + f"- Decision status: {_code(result.decision_status)}", + f"- Discovery status: {_code(result.discovery_status)}", + f"- Selected candidate: {_code(result.selected_candidate_id or 'none; shortlist entries are leads')}", f"- Reason: {_code(result.recommendation_reason)}", f"- Query: {_code(result.query)}", f"- Started: {_code(result.started_at)}", f"- Completed: {_code(result.completed_at)}", + f"- Displayed / evaluated candidates: {len(result.candidates)} / {len(_all_candidates(result))}", + f"- Core query: {_code(result.query_plan.get('core_query', result.query))}", + f"- Requested constraints: {_code('; '.join(result.query_plan.get('constraints', [])) or 'none extracted')}", + f"- Incomplete evidence: {_code('; '.join(result.incomplete_reasons) or 'none')}", "", "## Ranked candidates", "", - "| # | Score | Coverage | Candidate | Source | Feature | Activity | Dependencies | Security | Tests | Portability | Reuse | Adoption | Recommendation | Status |", - "|---:|---:|---:|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---|---|", + "| # | Score | Decision score | Coverage | Candidate | Source | Feature | Activity | Dependencies | Security | Tests | Portability | Reuse | Adoption | Recommendation | Status |", + "|---:|---:|---:|---:|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---|---|", ] for index, candidate in enumerate(result.candidates, 1): lines.append( @@ -70,6 +111,7 @@ def render_discovery_markdown(result: DiscoveryResult) -> str: [ str(index), f"{candidate.score.total if candidate.score else 0:.2f}", + f"{candidate.score.decision_score if candidate.score else 0:.2f}", f"{candidate.score.coverage if candidate.score else 0:.2f}", _code(candidate.name), _code(candidate.source), @@ -87,8 +129,13 @@ def render_discovery_markdown(result: DiscoveryResult) -> str: ) + " |" ) + lines.extend(["", "## Inspection allocation", ""]) + for allocation in result.inspection_decisions: + lines.append(f"- {_code(allocation['candidate'])}: {_code(allocation['reason'])}; status={_code(allocation['status'])}") + if not result.inspection_decisions: + lines.append("No inspection slots used; candidates remain unverified leads.") lines.extend(["", "## Candidate evidence", ""]) - for index, candidate in enumerate(result.candidates, 1): + for index, candidate in enumerate(_all_candidates(result), 1): lines.extend( [ f"### {index}. {_code(candidate.name)}", @@ -100,6 +147,8 @@ def render_discovery_markdown(result: DiscoveryResult) -> str: f"- Version: {_code(candidate.version or 'unknown')}", f"- Canonical identity: {_code(candidate.canonical_id or 'unknown')}", f"- License: {_code(candidate.license or 'unknown')}", + f"- Normalized license: {_code(candidate.normalized_license or 'unknown')}", + f"- Requested constraint checks: {_code('; '.join(candidate.constraint_checks) or 'none')}", f"- Activity: {_code(candidate.updated_at or candidate.published_at or 'unknown')}", f"- Dependencies: {_code(candidate.dependency_count if candidate.dependency_count is not None else 'unknown')}", f"- Stars / forks / watchers: {_code(' / '.join((_observed_number(candidate.stars), _observed_number(candidate.forks), _observed_number(candidate.watchers))))}", @@ -142,6 +191,8 @@ def render_discovery_markdown(result: DiscoveryResult) -> str: ) if source.error: lines.append(f" - Error: {_code(source.error)}") + if source.queries: + lines.append(f" - Queries: {_code('; '.join(source.queries))}") for warning in source.warnings: lines.append(f" - Warning: {_code(warning)}") lines.extend(["", "## Limitations", ""]) @@ -155,7 +206,7 @@ def write_discovery(result: DiscoveryResult, output_dir: Path) -> Path: receipt = atomic_write(output_dir / "discovery.md", render_discovery_markdown(result)) candidates_dir = output_dir / "candidates" candidates_dir.mkdir(exist_ok=True) - for index, candidate in enumerate(result.candidates, 1): + for index, candidate in enumerate(_all_candidates(result), 1): filename = re.sub(r"[^A-Za-z0-9_.-]+", "-", candidate.name).strip("-.")[:80] or "candidate" atomic_write( candidates_dir / f"{index:02d}-{filename}.json", diff --git a/src/brief2ship/discovery_scoring.py b/src/brief2ship/discovery_scoring.py index c2830c6..62ebb36 100644 --- a/src/brief2ship/discovery_scoring.py +++ b/src/brief2ship/discovery_scoring.py @@ -6,7 +6,9 @@ import re from datetime import datetime, timezone +from .discovery_licenses import is_permissive_license, normalize_license, license_body_match from .discovery_models import Candidate, ScoreBreakdown +from .discovery_query import plan_query _COMPONENT_MAX = { "feature_match": 25.0, @@ -28,19 +30,10 @@ "reuse_readiness": 0.15, "adoption_health": 0.05, } -_PERMISSIVE_LICENSES = { - "apache-2.0", - "apache 2.0", - "mit", - "bsd-2-clause", - "bsd-3-clause", - "isc", - "unlicense", - "0bsd", - "mit license", - "apache license 2.0", -} _STOPWORDS = {"a", "an", "and", "for", "of", "or", "the", "to", "with"} +_REUSE_RECOMMENDATIONS = frozenset({"use-as-library", "fork", "selective-reuse"}) +_FEATURE_RELEVANCE_THRESHOLD = 8.0 +_SOURCE_RANK_CAP = 20 def tokenize(value: str) -> set[str]: @@ -51,18 +44,6 @@ def tokenize(value: str) -> set[str]: } -def _permissive_license(value: str | None) -> bool: - normalized = (value or "").strip().lower().replace("_", "-") - if normalized in _PERMISSIVE_LICENSES: - return True - parts = [ - part.strip(" ()") - for part in re.split(r"\s+(?:or|and)\s+|,", normalized) - if part.strip(" ()") - ] - return bool(parts) and all(part in _PERMISSIVE_LICENSES for part in parts) - - def _parse_date(value: str | None) -> datetime | None: if not value: return None @@ -112,6 +93,7 @@ def _feature_score(query: str, candidate: Candidate) -> tuple[float, list[str]]: + identity_bonus, ) return score, [ + f"core relevance query={query}", f"name token coverage={name_overlap:.2f}", f"description token coverage={description_overlap:.2f}", f"topic token coverage={topic_overlap:.2f}", @@ -120,6 +102,15 @@ def _feature_score(query: str, candidate: Candidate) -> tuple[float, list[str]]: ] +def _record_constraint_checks(candidate: Candidate, constraints: tuple[str, ...]) -> None: + """Keep requested constraints explicit until a later gate verifies them.""" + + candidate.constraint_checks = [ + f"requested constraint unverified: {constraint}" + for constraint in dict.fromkeys(constraints) + ] + + def _maintenance_score(candidate: Candidate, now: datetime) -> tuple[float, list[str]]: if candidate.archived: return 0.0, ["repository is archived"] @@ -146,7 +137,7 @@ def _maintenance_score(candidate: Candidate, now: datetime) -> tuple[float, list def _dependency_score(candidate: Candidate) -> tuple[float, list[str]]: count = candidate.dependency_count - if count is None and candidate.inspection: + if count is None and candidate.inspection and candidate.source in {"github", "local"}: count = candidate.inspection.dependency_count if count is None: return 5.0, ["dependency count unknown; neutral score"] @@ -173,11 +164,10 @@ def _security_score(candidate: Candidate) -> tuple[float, list[str]]: else: vulnerability_points = 3.0 evidence.append("OSV status unknown; partial neutral score") - license_value = (candidate.license or "").strip().lower() - if _permissive_license(license_value): + if is_permissive_license(candidate.normalized_license): license_points = 4.0 - evidence.append(f"permissive license={candidate.license}") - elif license_value: + evidence.append(f"permissive license={candidate.normalized_license}") + elif candidate.license: license_points = 2.0 evidence.append(f"license requires review={candidate.license}") else: @@ -247,9 +237,9 @@ def _reuse_score(candidate: Candidate) -> tuple[float, list[str]]: if candidate.description: score += 1.0 evidence.append("description present") - if candidate.license: + if is_permissive_license(candidate.normalized_license): score += 1.0 - evidence.append("license declared") + evidence.append(f"normalized permissive license={candidate.normalized_license}") if inspection: if inspection.docs_files: score += 2.0 @@ -325,7 +315,9 @@ def recommend(candidate: Candidate) -> str: feature = candidate.score.components["feature_match"] security = candidate.score.components["security_posture"] license_missing = not bool((candidate.license or "").strip()) - license_incompatible = not license_missing and not _permissive_license(candidate.license) + license_incompatible = not license_missing and not is_permissive_license( + candidate.normalized_license + ) receipt = candidate.inspection.test_receipt if candidate.inspection else None if receipt and receipt.status in {"failed", "timeout", "oom", "signaled", "zero_tests"}: return "reject" @@ -357,13 +349,14 @@ def _score_coverage(candidate: Candidate) -> dict[str, float]: inspection_complete = bool(candidate.inspection and candidate.inspection.status == "inspected") inspection_dependency_known = bool( inspection_complete + and candidate.source in {"github", "local"} and candidate.inspection and candidate.inspection.dependency_count is not None ) test_receipt = candidate.inspection.test_receipt if candidate.inspection else None security_coverage = ( (0.60 if candidate.vulnerabilities_checked else 0.0) - + (0.25 if candidate.license else 0.0) + + (0.25 if candidate.normalized_license else 0.0) + (0.15 if candidate.security_policy is not None else 0.0) ) return { @@ -373,7 +366,7 @@ def _score_coverage(candidate: Candidate) -> dict[str, float]: "security_posture": security_coverage, "test_quality": 1.0 if inspection_complete else 0.35 if candidate.test_signals else 0.0, "portability": 1.0 if test_receipt and test_receipt.status == "passed" else 0.70 if candidate.portability_signals else 0.40 if candidate.language else 0.0, - "reuse_readiness": 1.0 if inspection_complete else 0.60 if candidate.repository_url and candidate.license else 0.30 if candidate.repository_url else 0.0, + "reuse_readiness": 1.0 if inspection_complete else 0.60 if candidate.repository_url and candidate.normalized_license else 0.30 if candidate.repository_url else 0.0, "adoption_health": 1.0 if any( value is not None @@ -393,6 +386,10 @@ def _score_coverage(candidate: Candidate) -> dict[str, float]: def _recommendation_receipt(candidate: Candidate) -> None: blockers: list[str] = [] checks: list[str] = [] + if candidate.license_review_required: + checks.append("complete MIT body recognized; review free-form copyright and surrounding text before reuse") + if candidate.inspection and candidate.source in {"pypi", "npm", "crates", "huggingface"}: + checks.append("verify package-specific source, tests and license; static inspection covers the repository, not a proven package subtree") if candidate.archived: blockers.append("repository is archived") if candidate.deprecated: @@ -403,7 +400,7 @@ def _recommendation_receipt(candidate: Candidate) -> None: blockers.append("artifact is disabled") if not candidate.license: blockers.append("license missing or ambiguous") - elif not _permissive_license(candidate.license): + elif not is_permissive_license(candidate.normalized_license): blockers.append("license is outside the default permissive allowlist") if candidate.vulnerabilities: blockers.append("OSV findings require severity and remediation review") @@ -417,11 +414,12 @@ def _recommendation_receipt(candidate: Candidate) -> None: checks.append("exact package-version OSV evidence unavailable") if not receipt or receipt.status != "passed": checks.append("authorized sandbox test pass unavailable") + checks.extend(candidate.constraint_checks) candidate.hard_blockers = blockers - candidate.required_checks = checks + candidate.required_checks = list(dict.fromkeys(checks)) if blockers: candidate.recommendation_status = "blocked" - elif candidate.recommendation in {"use-as-library", "fork", "selective-reuse"}: + elif candidate.recommendation in _REUSE_RECOMMENDATIONS: candidate.recommendation_status = "ready" if not checks else "provisional" else: candidate.recommendation_status = "not-selected" @@ -439,8 +437,13 @@ def _canonical_id(candidate: Candidate) -> str: def score_candidate(query: str, candidate: Candidate, *, now: datetime | None = None) -> ScoreBreakdown: current = now or datetime.now(timezone.utc) + query_plan = plan_query(query) + candidate.normalized_license = normalize_license(candidate.license, metadata=candidate.license_kind != "file") + candidate.license_body_match = license_body_match(candidate.license) + candidate.license_review_required = candidate.license_body_match is not None and candidate.normalized_license is None + _record_constraint_checks(candidate, query_plan.constraints) scorers = { - "feature_match": lambda: _feature_score(query, candidate), + "feature_match": lambda: _feature_score(query_plan.core_query, candidate), "maintenance_activity": lambda: _maintenance_score(candidate, current), "dependency_weight": lambda: _dependency_score(candidate), "security_posture": lambda: _security_score(candidate), @@ -463,14 +466,22 @@ def score_candidate(query: str, candidate: Candidate, *, now: datetime | None = _COMPONENT_MAX[name] * _UNKNOWN_PENALTY[name] * (1.0 - coverage_by_component[name]) for name in _COMPONENT_MAX ) + total = round(sum(components.values()), 2) + decision_score = round(max(0.0, total - unknown_cost), 2) for name, value in coverage_by_component.items(): evidence[name].append(f"evidence coverage={value:.2f}") + evidence["decision_score"] = [ + f"raw total={total:.2f}", + f"unknown evidence cost={unknown_cost:.2f}", + f"decision score=max(0, raw total - unknown evidence cost)={decision_score:.2f}", + ] breakdown = ScoreBreakdown( - round(sum(components.values()), 2), - components, - evidence, + total=total, + components=components, + evidence=evidence, coverage=round(coverage, 4), unknown_cost=round(unknown_cost, 2), + decision_score=decision_score, ) candidate.score = breakdown candidate.canonical_id = _canonical_id(candidate) @@ -479,14 +490,71 @@ def score_candidate(query: str, candidate: Candidate, *, now: datetime | None = return breakdown +def candidate_rank_key( + candidate: Candidate, +) -> tuple[int, float, float, int, float, float, str, int, str, str]: + """Return a deterministic evidence-aware ordering key. + + Provider ``raw_relevance`` values are intentionally excluded because their + scales are not comparable. ``source_rank`` is capped and considered only + after the source name, so it can order otherwise-tied results from the same + provider without becoming a cross-provider score. + """ + + score = candidate.score + feature = score.components.get("feature_match", 0.0) if score else 0.0 + decision = score.decision_score if score else 0.0 + total = score.total if score else 0.0 + coverage = score.coverage if score else 0.0 + blocked = ( + bool(candidate.hard_blockers) + or candidate.recommendation == "reject" + or candidate.recommendation_status == "blocked" + ) + relevant = feature >= _FEATURE_RELEVANCE_THRESHOLD + usable = ( + candidate.recommendation in _REUSE_RECOMMENDATIONS + and candidate.recommendation_status in {"ready", "provisional"} + ) + if blocked: + eligibility_tier = 3 + elif relevant and usable: + eligibility_tier = 0 + elif relevant: + eligibility_tier = 1 + else: + eligibility_tier = 2 + + if candidate.recommendation_status == "ready": + confidence_tier = 0 + elif candidate.inspection and candidate.inspection.status == "inspected": + confidence_tier = 1 + elif candidate.recommendation_status == "provisional": + confidence_tier = 2 + else: + confidence_tier = 3 + + source_rank = candidate.source_rank + bounded_source_rank = ( + min(_SOURCE_RANK_CAP, source_rank) + if isinstance(source_rank, int) and source_rank > 0 + else _SOURCE_RANK_CAP + 1 + ) + return ( + eligibility_tier, + -decision, + -feature, + confidence_tier, + -coverage, + -total, + candidate.source, + bounded_source_rank, + candidate.name.lower(), + candidate.url, + ) + + def rank_candidates(query: str, candidates: list[Candidate], *, now: datetime | None = None) -> list[Candidate]: for candidate in candidates: score_candidate(query, candidate, now=now) - return sorted( - candidates, - key=lambda item: ( - -(item.score.total if item.score else 0), - item.source, - item.name.lower(), - ), - ) + return sorted(candidates, key=candidate_rank_key) diff --git a/src/brief2ship/models.py b/src/brief2ship/models.py index 52bc24a..be38be9 100644 --- a/src/brief2ship/models.py +++ b/src/brief2ship/models.py @@ -6,7 +6,7 @@ from typing import Any -DEFAULT_USER_AGENT = "Brief2ShipBot/0.6.2 (+https://github.com/five0nit/brief2ship)" +DEFAULT_USER_AGENT = "Brief2ShipBot/0.7.0 (+https://github.com/five0nit/brief2ship)" @dataclass(frozen=True) diff --git a/tests/fixtures/discovery-quality-v1.json b/tests/fixtures/discovery-quality-v1.json new file mode 100644 index 0000000..7be4a67 --- /dev/null +++ b/tests/fixtures/discovery-quality-v1.json @@ -0,0 +1,151 @@ +{ + "schema": "brief2ship-quality-fixtures-v1", + "provenance": "Synthetic regression scenarios, agent-authored; not live provider data or a human-reviewed accuracy benchmark. Candidate names and evidence are fictional.", + "clock": "2026-09-01T00:00:00+00:00", + "cases": [ + { + "id": "web-scraper-short", + "query": "web scraper", + "expected": "fixture/arachnid", + "description": "Robots-aware web scraper for public pages" + }, + { + "id": "web-scraper-constraints", + "query": "I need a web scraper for Windows with no cloud and no paid APIs", + "expected": "fixture/arachnid", + "description": "Robots-aware web scraper for public pages" + }, + { + "id": "sqlite-short", + "query": "sqlite database", + "expected": "fixture/granite", + "description": "Embedded sqlite database toolkit" + }, + { + "id": "sqlite-constraints", + "query": "I need a sqlite database for Windows with no cloud and no paid APIs", + "expected": "fixture/granite", + "description": "Embedded sqlite database toolkit" + }, + { + "id": "pdf-short", + "query": "pdf text extraction", + "expected": "fixture/folio", + "description": "PDF text extraction library" + }, + { + "id": "pdf-constraints", + "query": "I need a pdf text extraction for Windows with no cloud and no paid APIs", + "expected": "fixture/folio", + "description": "PDF text extraction library" + }, + { + "id": "calendar-short", + "query": "calendar event scheduling", + "expected": "fixture/almanac", + "description": "Calendar event scheduling library" + }, + { + "id": "calendar-constraints", + "query": "I need a calendar event scheduling for Windows with no cloud and no paid APIs", + "expected": "fixture/almanac", + "description": "Calendar event scheduling library" + }, + { + "id": "markdown-short", + "query": "markdown renderer", + "expected": "fixture/quill", + "description": "Markdown renderer for documents" + }, + { + "id": "markdown-constraints", + "query": "I need a markdown renderer for Windows with no cloud and no paid APIs", + "expected": "fixture/quill", + "description": "Markdown renderer for documents" + }, + { + "id": "audio-short", + "query": "audio transcription", + "expected": "fixture/echo", + "description": "Audio transcription toolkit" + }, + { + "id": "audio-constraints", + "query": "I need a audio transcription for Windows with no cloud and no paid APIs", + "expected": "fixture/echo", + "description": "Audio transcription toolkit" + }, + { + "id": "android-short", + "query": "android launcher", + "expected": "fixture/orbit", + "description": "Android launcher application" + }, + { + "id": "android-constraints", + "query": "I need a android launcher for Windows with no cloud and no paid APIs", + "expected": "fixture/orbit", + "description": "Android launcher application" + }, + { + "id": "json-short", + "query": "json schema validation", + "expected": "fixture/contract", + "description": "JSON schema validation library" + }, + { + "id": "json-constraints", + "query": "I need a json schema validation for Windows with no cloud and no paid APIs", + "expected": "fixture/contract", + "description": "JSON schema validation library" + }, + { + "id": "printer-short", + "query": "network printer discovery", + "expected": "fixture/beacon", + "description": "Network printer discovery utility" + }, + { + "id": "printer-constraints", + "query": "I need a network printer discovery for Windows with no cloud and no paid APIs", + "expected": "fixture/beacon", + "description": "Network printer discovery utility" + }, + { + "id": "shell-short", + "query": "command shell", + "expected": "fixture/conch", + "description": "Interactive command shell" + }, + { + "id": "shell-constraints", + "query": "I need a command shell for Windows with no cloud and no paid APIs", + "expected": "fixture/conch", + "description": "Interactive command shell" + }, + { + "id": "catalogue-short", + "query": "project catalogue", + "expected": "fixture/atlas", + "description": "Local project catalogue tool" + }, + { + "id": "catalogue-constraints", + "query": "I need a project catalogue for Windows with no cloud and no paid APIs", + "expected": "fixture/atlas", + "description": "Local project catalogue tool" + }, + { + "id": "csv-short", + "query": "csv data analysis", + "expected": "fixture/tabular", + "description": "CSV data analysis library" + }, + { + "id": "csv-constraints", + "query": "I need a csv data analysis for Windows with no cloud and no paid APIs", + "expected": "fixture/tabular", + "description": "CSV data analysis library" + } + ] +} diff --git a/tests/test_cli.py b/tests/test_cli.py index 7472770..6ad26d1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -150,6 +150,8 @@ def test_discover_writes_ranked_receipt(self): sources=[SourceReceipt("github", "ok", 1, 1)], overall_recommendation=candidate.recommendation, recommendation_reason="test", + decision_status="provisional", + discovery_status="complete", ) with tempfile.TemporaryDirectory() as temporary, patch( "brief2ship.cli.discover_candidates", return_value=result @@ -186,6 +188,8 @@ def fake_discover(query, config, *, output_dir): sources=[SourceReceipt("local", "ok", 1, 1)], overall_recommendation="selective-reuse", recommendation_reason="test", + decision_status="provisional", + discovery_status="complete", ) with tempfile.TemporaryDirectory() as temporary, patch( diff --git a/tests/test_discovery_decisions.py b/tests/test_discovery_decisions.py new file mode 100644 index 0000000..0cd89b6 --- /dev/null +++ b/tests/test_discovery_decisions.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import Mock, patch + +from brief2ship.discovery import discover +from brief2ship.discovery_models import Candidate, DiscoveryConfig, InspectionResult, SourceReceipt +from brief2ship.discovery_render import write_discovery + + +def candidate(name: str = "web-scraper", description: str = "web scraper") -> Candidate: + return Candidate( + source="github", name=f"example/{name}", + url=f"https://github.com/example/{name}", + repository_url=f"https://github.com/example/{name}", + description=description, license="MIT", + updated_at=datetime.now(timezone.utc).isoformat(), + dependency_count=0, language="Python", + ) + + +class DecisionTests(unittest.TestCase): + def run_discovery(self, candidates, *, limit=5, inspect_top=0, failure=False, partial=False): + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + github = Mock(return_value=( + candidates, SourceReceipt("github", "failed" if failure else "ok", 5, + returned=len(candidates), error="fixture outage" if failure else None), + )) + providers = {"github": github} + sources = ("github",) + if partial: + providers["npm"] = Mock(return_value=([], SourceReceipt("npm", "failed", 5, error="fixture outage"))) + sources += ("npm",) + inspector = Mock() + inspector.inspect.side_effect = lambda item, run_tests=False: InspectionResult( + repository_url=item.repository_url or "", status="inspected", commit="fixture-commit", + license=item.license, dependency_count=0, + ) + with patch.dict("brief2ship.discovery.PROVIDERS", providers), patch( + "brief2ship.discovery.RepositoryInspector", return_value=inspector, + ): + result = discover("web scraper", DiscoveryConfig( + sources=sources, per_source=5, limit=limit, inspect_top=inspect_top, + ), output_dir=root / "receipt") + return result, root / "receipt" + + def test_failed_sources_never_authorize_clean_build(self): + result, _ = self.run_discovery([], failure=True) + self.assertEqual("inconclusive", result.overall_recommendation) + self.assertEqual("inconclusive", result.decision_status) + self.assertEqual("failed", result.discovery_status) + self.assertTrue(result.incomplete_reasons) + + def test_successful_empty_search_is_not_proof_of_no_reusable_code(self): + result, _ = self.run_discovery([]) + self.assertEqual("complete", result.discovery_status) + self.assertEqual("inconclusive", result.overall_recommendation) + self.assertIsNone(result.selected_candidate_id) + + def test_uninspected_lead_is_not_run_level_reuse_choice(self): + result, _ = self.run_discovery([candidate()]) + self.assertEqual("inconclusive", result.overall_recommendation) + self.assertIsNone(result.selected_candidate_id) + self.assertTrue(any("inspection" in value for value in result.incomplete_reasons)) + + def test_output_limit_does_not_change_decision_or_drop_evidence(self): + decisions = [] + for limit in (1, 2): + irrelevant = candidate("aaa-unrelated", "healthy utility") + irrelevant.stars = 1_000_000 + irrelevant.security_policy = True + result, _ = self.run_discovery([irrelevant, candidate()], limit=limit, inspect_top=1) + decisions.append((result.overall_recommendation, result.selected_candidate_id)) + self.assertEqual(2, len(result.evaluated_candidates)) + self.assertEqual(limit, len(result.candidates)) + self.assertEqual("selective-reuse", result.overall_recommendation) + self.assertEqual("provisional", result.decision_status) + self.assertEqual(1, len(result.inspection_decisions)) + self.assertEqual(decisions[0], decisions[1]) + + def test_partial_provider_failure_is_not_hidden_by_reusable_candidate(self): + result, _ = self.run_discovery([candidate()], inspect_top=1, partial=True) + self.assertEqual("partial", result.discovery_status) + self.assertEqual("inconclusive", result.overall_recommendation) + self.assertTrue(any("npm" in value for value in result.incomplete_reasons)) + + def test_full_json_markdown_and_artifacts_retain_all_candidates(self): + result, output = self.run_discovery( + [candidate(), candidate("aaa-unrelated", "utility"), candidate("bbb-unrelated", "utility")], + limit=1, inspect_top=1, + ) + receipt = write_discovery(result, output) + payload = json.loads((output / "discovery.json").read_text()) + self.assertEqual(1, len(payload["candidates"])) + self.assertEqual(3, len(payload["evaluated_candidates"])) + self.assertEqual(3, len(list((output / "candidates").glob("*.json")))) + markdown = receipt.read_text() + self.assertIn("example/aaa-unrelated", markdown) + self.assertIn("Inspection allocation", markdown) + self.assertIn("provisional", markdown) + + def test_unrecognized_license_text_is_not_supported_negative_evidence(self): + item = candidate("unrelated", "utility") + item.license = "unknown; contact author" + result, _ = self.run_discovery([item], inspect_top=1) + self.assertEqual("inconclusive", result.overall_recommendation) + + def test_ready_candidate_maps_to_complete_run_status(self): + from brief2ship.discovery_decision import decide + item = candidate() + item.recommendation = "selective-reuse" + item.recommendation_status = "ready" + item.inspection = InspectionResult(repository_url=item.repository_url or "", status="inspected") + outcome = decide([item], [SourceReceipt("github", "ok", 1, returned=1)]) + self.assertEqual("complete", outcome.status) + + def test_clean_build_requires_completed_negative_inspection(self): + result, _ = self.run_discovery([candidate("unrelated", "utility")], inspect_top=1) + self.assertEqual("build-clean", result.overall_recommendation) + self.assertEqual("complete", result.decision_status) + self.assertIsNone(result.selected_candidate_id) + + def test_failed_inspection_retained_and_does_not_authorize_clean_build(self): + with tempfile.TemporaryDirectory() as root, patch.dict( + "brief2ship.discovery.PROVIDERS", {"github": Mock(return_value=( + [candidate()], SourceReceipt("github", "ok", 1, returned=1), + ))}, + ), patch("brief2ship.discovery.RepositoryInspector") as inspector: + inspector.return_value.inspect.return_value = InspectionResult( + repository_url="https://github.com/example/web-scraper", status="blocked", + warnings=["fixture unavailable"], + ) + result = discover("web scraper", DiscoveryConfig(sources=("github",), inspect_top=1), output_dir=Path(root)) + self.assertEqual("inconclusive", result.overall_recommendation) + self.assertEqual("blocked", result.inspection_decisions[0]["status"]) + inspection = result.evaluated_candidates[0].inspection + self.assertIsNotNone(inspection) + self.assertEqual("blocked", inspection.status if inspection else None) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_discovery_evidence_scope.py b/tests/test_discovery_evidence_scope.py new file mode 100644 index 0000000..ae099b7 --- /dev/null +++ b/tests/test_discovery_evidence_scope.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import unittest + +from brief2ship.discovery import deduplicate_candidates +from brief2ship.discovery_inspection import RepositoryInspector +from brief2ship.discovery_models import Candidate, InspectionResult +from brief2ship.discovery_scoring import score_candidate + + +class EvidenceScopeTests(unittest.TestCase): + def package(self): + return Candidate("npm", "subpackage", "https://www.npmjs.com/package/subpackage", + repository_url="https://github.com/example/monorepo", version="1.0.0") + + def test_repository_merge_cannot_relabel_unknown_package_dependencies(self): + package = self.package() + repo = Candidate("github", "example/monorepo", "https://github.com/example/monorepo", + repository_url=package.repository_url, dependency_count=0, + stars=123, description="repository-wide capabilities") + merged = deduplicate_candidates([package, repo])[0] + self.assertIsNone(merged.dependency_count) + self.assertEqual("", merged.description) + self.assertEqual(123, merged.stars) + self.assertEqual(0, merged.repository_evidence["dependency_count"]) + + def test_repository_inspection_does_not_overwrite_package_dependencies(self): + package = self.package() + package.dependency_count = 3 + inspection = InspectionResult(repository_url=package.repository_url or "", status="inspected", dependency_count=70) + package.inspection = inspection + RepositoryInspector._apply_inspection_evidence(package, inspection) + self.assertEqual(3, package.dependency_count) + self.assertEqual(70, package.inspection.dependency_count) + + def test_package_scoring_cannot_promote_unknown_using_repository_count(self): + package = self.package() + package.inspection = InspectionResult(repository_url=package.repository_url or "", status="inspected", dependency_count=0) + RepositoryInspector._apply_inspection_evidence(package, package.inspection) + score = score_candidate("subpackage", package) + self.assertIsNone(package.dependency_count) + self.assertEqual(5, score.components["dependency_weight"]) + self.assertTrue(any("unknown" in line for line in score.evidence["dependency_weight"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_discovery_inspection.py b/tests/test_discovery_inspection.py index 2111099..ed4c09f 100644 --- a/tests/test_discovery_inspection.py +++ b/tests/test_discovery_inspection.py @@ -58,7 +58,7 @@ def _fixture(self, root: Path) -> None: (root / "pyproject.toml").write_text( '[project]\nname="fixture"\ndependencies=["one", "two"]\n', encoding="utf-8" ) - (root / "LICENSE").write_text("MIT License\n", encoding="utf-8") + (root / "LICENSE").write_bytes((Path(__file__).resolve().parents[1] / "LICENSE").read_bytes()) (root / "README.md").write_text("fixture", encoding="utf-8") (root / "src/tool.py").write_text("VALUE = 1\n", encoding="utf-8") (root / "tests/test_ok.py").write_text( @@ -93,7 +93,7 @@ def test_static_inspection_collects_real_evidence(self): self._fixture(root) result = inspect_tree(root, "https://github.com/example/fixture") self.assertEqual(result.status, "inspected") - self.assertEqual(result.license, "MIT") + self.assertEqual(result.license, (Path(__file__).resolve().parents[1] / "LICENSE").read_bytes().decode("utf-8")) self.assertEqual(result.dependency_count, 2) self.assertIn("pyproject.toml", result.manifest_files) self.assertEqual(len(result.test_files), 2) @@ -239,7 +239,7 @@ def fake_run(command, **kwargs): '[project]\nname="fixture"\ndependencies=[]\n', encoding="utf-8" ) (target / "README.md").write_text("fixture scraper", encoding="utf-8") - (target / "LICENSE").write_text("MIT License", encoding="utf-8") + (target / "LICENSE").write_bytes((Path(__file__).resolve().parents[1] / "LICENSE").read_bytes()) (target / "tests/test_ok.py").write_text("import unittest\n", encoding="utf-8") return subprocess.CompletedProcess(command, 0, "", "") return subprocess.CompletedProcess(command, 0, "a" * 40 + "\n", "") @@ -255,7 +255,7 @@ def fake_run(command, **kwargs): ): result = RepositoryInspector(InspectorClient(), Path(temporary)).inspect(candidate) self.assertEqual("inspected", result.status) - self.assertEqual("MIT", candidate.license) + self.assertEqual((Path(__file__).resolve().parents[1] / "LICENSE").read_bytes().decode("utf-8"), candidate.license) self.assertTrue(candidate.security_policy) clone = next(command for command in calls if "clone" in command) self.assertIn("--depth=1", clone) diff --git a/tests/test_discovery_integration.py b/tests/test_discovery_integration.py index 3b6c741..8ac6a2b 100644 --- a/tests/test_discovery_integration.py +++ b/tests/test_discovery_integration.py @@ -95,15 +95,15 @@ def test_all_sources_produce_ranked_deterministic_receipts(self): self.assertEqual({source.source for source in result.sources}, {"github", "pypi", "npm", "crates", "huggingface"}) self.assertGreaterEqual(len(result.candidates), 5) self.assertEqual(payload["query"], "safe web scraper") - self.assertEqual(payload["schema_version"], "brief2ship-discovery-v1") - self.assertEqual(payload["scoring_contract"], "brief2ship-score-v1") + self.assertEqual(payload["schema_version"], "brief2ship-discovery-v2") + self.assertEqual(payload["scoring_contract"], "brief2ship-score-v2") self.assertEqual(payload["sandbox_policy"], "brief2ship-bwrap-v2") self.assertEqual(payload["config"]["total_timeout"], 180.0) self.assertIn("coverage", payload["candidates"][0]["score"]) self.assertIn("recommendation_status", payload["candidates"][0]) self.assertIn("Ranked candidates", markdown) - scores = [candidate.score.total if candidate.score else -1 for candidate in result.candidates] - self.assertEqual(scores, sorted(scores, reverse=True)) + self.assertEqual("inconclusive", result.decision_status) + self.assertEqual(len(result.candidates), len(result.evaluated_candidates)) def test_output_directory_must_start_empty(self): with tempfile.TemporaryDirectory() as temporary: @@ -232,7 +232,7 @@ def test_local_only_discovery_inspects_existing_project_without_network(self): "# Local Agent Starter\n", encoding="utf-8", ) - (project / "LICENSE").write_text("MIT License\n", encoding="utf-8") + (project / "LICENSE").write_text((Path(__file__).resolve().parents[1] / "LICENSE").read_text(encoding="utf-8"), encoding="utf-8") (project / "tests/test_one.py").write_text("VALUE = 1\n", encoding="utf-8") result = discover( diff --git a/tests/test_discovery_license_sources.py b/tests/test_discovery_license_sources.py new file mode 100644 index 0000000..b5581af --- /dev/null +++ b/tests/test_discovery_license_sources.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from brief2ship.discovery_inspection import RepositoryInspector, inspect_tree +from brief2ship.discovery_local import search_local +from brief2ship.discovery_models import Candidate +from brief2ship.discovery_scoring import score_candidate + + +class LicenseSourceTests(unittest.TestCase): + def test_inspection_preserves_modified_file_and_overrides_metadata_claim(self): + raw = "MIT License\n\nCommercial use is prohibited.\n" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "LICENSE").write_bytes(raw.encode("utf-8")) + inspection = inspect_tree(root, root.as_uri()) + item = Candidate("github", "example/web-scraper", "https://github.com/example/web-scraper", license="MIT") + item.inspection = inspection + RepositoryInspector._apply_inspection_evidence(item, inspection) + score_candidate("web scraper", item) + self.assertEqual(raw, inspection.license) + self.assertEqual(raw, item.license) + self.assertIsNone(item.normalized_license) + self.assertEqual("reject", item.recommendation) + + def test_both_license_sources_preserve_lf_and_crlf_bytes(self): + for newline in ("\n", "\r\n"): + with self.subTest(newline=repr(newline)), tempfile.TemporaryDirectory() as directory: + root = Path(directory) / "web-scraper" + root.mkdir() + raw = newline.join(("MIT License", "", "Commercial use is prohibited.", "")) + (root / "pyproject.toml").write_text( + '[project]\nname="web-scraper"\nversion="1.0"\n', encoding="utf-8" + ) + (root / "LICENSE").write_bytes(raw.encode("utf-8")) + inspected = inspect_tree(root, root.resolve().as_uri()) + candidates, _ = search_local("web scraper", (str(root),), limit=3) + self.assertEqual(1, len(candidates)) + self.assertEqual(raw, inspected.license) + self.assertEqual(raw, candidates[0].license) + score_candidate("web scraper", candidates[0]) + self.assertIsNone(candidates[0].normalized_license) + self.assertEqual("reject", candidates[0].recommendation) + + def test_local_title_only_license_file_is_not_a_canonical_grant(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) / "web-scraper" + root.mkdir() + (root / "pyproject.toml").write_text('[project]\nname="web-scraper"\nversion="1.0"\n', encoding="utf-8") + (root / "LICENSE").write_text("MIT License\n", encoding="utf-8") + candidates, _ = search_local("web scraper", (str(root),), limit=3) + self.assertEqual(1, len(candidates)) + score_candidate("web scraper", candidates[0]) + self.assertIsNone(candidates[0].normalized_license) + self.assertEqual("reject", candidates[0].recommendation) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_discovery_licenses.py b/tests/test_discovery_licenses.py new file mode 100644 index 0000000..c881312 --- /dev/null +++ b/tests/test_discovery_licenses.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import unittest + +from brief2ship.discovery_licenses import normalize_license, license_body_match + + +MIT_BODY = '''Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.''' + + +class DiscoveryLicenseTests(unittest.TestCase): + def test_exact_permissive_aliases_are_canonicalized(self): + expected = { + "MIT": "MIT", + "MIT License": "MIT", + "Apache-2.0": "Apache-2.0", + "Apache License 2.0": "Apache-2.0", + "BSD-2-Clause": "BSD-2-Clause", + "BSD-3-Clause": "BSD-3-Clause", + "ISC": "ISC", + "Unlicense": "Unlicense", + "0BSD": "0BSD", + } + + for raw, canonical in expected.items(): + with self.subTest(raw=raw): + self.assertEqual(canonical, normalize_license(raw)) + + def test_canonical_mit_body_allows_titles_but_not_freeform_notices(self): + variants = ( + MIT_BODY, + f"MIT License\n\n{MIT_BODY}", + f"The MIT License (MIT)\r\n\r\n{MIT_BODY}", + ) + + for raw in variants: + with self.subTest(raw=raw[:40]): + self.assertEqual("MIT", normalize_license(raw)) + + def test_freeform_notices_are_recognized_but_never_authorize_reuse(self): + from brief2ship.discovery_scoring import rank_candidates + from brief2ship.discovery_models import Candidate, InspectionResult, SourceReceipt + from brief2ship.discovery_decision import decide + for notice in ( + "Copyright (c) 2026 Example Authors", + "Copyright © 2018-2026 Example, Inc.", + "Copyright 2026 Example Corp Commercial exploitation forbidden", + "Copyright 2026 Must Have Software Pty Ltd", + ): + with self.subTest(notice=notice): + raw = f"MIT License\n{notice}\n\n{MIT_BODY}" + self.assertIsNone(normalize_license(raw)) + self.assertEqual("MIT", license_body_match(raw)) + item = Candidate("local", "web-scraper", "file:///fixture", description="web scraper", license=raw, + license_kind="file", dependency_count=0, test_signals=["unit tests", "continuous integration"]) + item.inspection = InspectionResult(repository_url=item.url, status="inspected") + rank_candidates("web scraper", [item]) + self.assertTrue(item.license_review_required) + self.assertIsNone(item.normalized_license) + self.assertEqual("MIT", item.license_body_match) + self.assertEqual("inconclusive", decide([item], [SourceReceipt("local", "ok", requested=1)]).recommendation) + + def test_truncated_or_modified_mit_body_is_rejected(self): + variants = ( + MIT_BODY[:-20], + f"MIT License\nCopyright (c) 2026 Authors. Non-commercial use only.\n\n{MIT_BODY}", + f"{MIT_BODY}\n\nUse is prohibited for commercial purposes.", + MIT_BODY.replace("without restriction", "only for non-commercial use"), + ) + + for raw in variants: + with self.subTest(raw=raw[-60:]): + self.assertIsNone(normalize_license(raw)) + + def test_license_substrings_and_restrictive_expressions_are_not_accepted(self): + for raw in ( + "MIT with Commons Clause", + "MIT OR GPL-3.0-only", + "Custom MIT-style license", + "The software includes MIT-licensed components", + "GPL-3.0-only", + "", + ): + with self.subTest(raw=raw): + self.assertIsNone(normalize_license(raw)) + + def test_none_is_unknown(self): + self.assertIsNone(normalize_license(None)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_discovery_local.py b/tests/test_discovery_local.py index e9744bb..796e216 100644 --- a/tests/test_discovery_local.py +++ b/tests/test_discovery_local.py @@ -19,7 +19,7 @@ def _project(self, root: Path, name: str, readme: str) -> Path: encoding="utf-8", ) (project / "README.md").write_text(readme, encoding="utf-8") - (project / "LICENSE").write_text("MIT License\n", encoding="utf-8") + (project / "LICENSE").write_bytes((Path(__file__).resolve().parents[1] / "LICENSE").read_bytes()) (project / "SECURITY.md").write_text("# Security\n", encoding="utf-8") (project / "tests/test_smoke.py").write_text("VALUE = 1\n", encoding="utf-8") (project / ".git/config").write_text( @@ -49,7 +49,7 @@ def test_local_search_finds_and_describes_real_project(self): self.assertEqual(str(project.resolve()), candidates[0].local_path) self.assertEqual(project.resolve().as_uri(), candidates[0].url) self.assertEqual("https://github.com/example/telegram-agent-kit", candidates[0].repository_url) - self.assertEqual("MIT", candidates[0].license) + self.assertEqual((Path(__file__).resolve().parents[1] / "LICENSE").read_bytes().decode("utf-8"), candidates[0].license) self.assertEqual("Python", candidates[0].language) self.assertTrue(candidates[0].security_policy) self.assertIn("repository test files", candidates[0].test_signals) @@ -145,6 +145,7 @@ def test_local_scan_is_breadth_first_so_deep_sibling_cannot_starve_target(self): self.assertEqual([str(target.resolve())], [candidate.local_path for candidate in candidates]) self.assertTrue(any("stopped" in warning for warning in receipt.warnings)) + self.assertEqual("partial", receipt.status) def test_local_scan_round_robins_roots_under_global_directory_cap(self): with tempfile.TemporaryDirectory() as temporary: @@ -170,6 +171,7 @@ def test_local_scan_round_robins_roots_under_global_directory_cap(self): self.assertEqual([str(target.resolve())], [candidate.local_path for candidate in candidates]) self.assertTrue(any("stopped" in warning for warning in receipt.warnings)) + self.assertEqual("partial", receipt.status) def test_local_scan_caps_entries_per_directory(self): with tempfile.TemporaryDirectory() as temporary: diff --git a/tests/test_discovery_partial_sources.py b/tests/test_discovery_partial_sources.py new file mode 100644 index 0000000..e152a68 --- /dev/null +++ b/tests/test_discovery_partial_sources.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from brief2ship.discovery_http import DiscoverySourceError +from brief2ship.discovery_providers import search_npm, search_pypi, search_crates, search_huggingface +from tests.test_discovery_providers import FakeClient + + +class PartialSourceTests(unittest.TestCase): + def test_npm_failed_detail_is_partial_and_endpoint_is_retained(self): + client = FakeClient() + client.routes['https://registry.npmjs.org/-/v1/search?text=web+scraper&size=1'] = { + 'objects': [{'package': {'name': 'web-scraper', 'version': '1.0.0'}}], + } + detail = 'https://registry.npmjs.org/web-scraper' + client.routes[detail] = DiscoverySourceError('fixture outage') + candidates, receipt = search_npm('web scraper', 1, client) + self.assertEqual(1, len(candidates)) + self.assertEqual('partial', receipt.status) + self.assertIn(detail, receipt.endpoints) + self.assertIn('fixture outage', receipt.error) + self.assertIsNone(candidates[0].dependency_count) + + def test_huggingface_malformed_nested_fields_write_receipts(self): + import json + from brief2ship.discovery import discover + from brief2ship.discovery_models import DiscoveryConfig + from brief2ship.discovery_render import write_discovery + for bad in ({"cardData": "malformed"}, {"tags": {}}, {"likes": []}, {"gated": {}}): + with self.subTest(bad=bad), tempfile.TemporaryDirectory() as directory: + client = FakeClient() + for kind in ("models", "datasets", "spaces"): + client.routes[f"https://huggingface.co/api/{kind}?search=web+scraper&limit=1&full=true"] = [ + {"id": "fixture/item", **bad}, + ] + result = discover("web scraper", DiscoveryConfig(sources=("huggingface",), per_source=3), output_dir=Path(directory), client=client) + self.assertEqual("inconclusive", result.overall_recommendation) + self.assertEqual("partial", result.sources[0].status) + self.assertEqual(3, len(result.sources[0].endpoints)) + write_discovery(result, Path(directory)) + self.assertEqual("inconclusive", json.loads((Path(directory) / "discovery.json").read_text())["overall_recommendation"]) + + def test_pypi_failed_project_request_is_partial(self): + client = FakeClient() + detail = 'https://pypi.org/pypi/safe-web-scraper/json' + client.routes[detail] = DiscoverySourceError('fixture outage') + with tempfile.TemporaryDirectory() as root: + _, receipt = search_pypi('web scraper', 1, client, cache_dir=Path(root)) + self.assertEqual('partial', receipt.status) + self.assertIn(detail, receipt.endpoints) + self.assertIn('fixture outage', receipt.error) + + def test_crates_failed_dependency_request_is_partial(self): + client = FakeClient() + client.routes['https://crates.io/api/v1/crates?q=web+scraper&sort=relevance&per_page=1&page=1'] = { + 'crates': [{'id': 'web-scraper', 'max_stable_version': '1.0.0'}], + } + detail = 'https://crates.io/api/v1/crates/web-scraper/1.0.0/dependencies' + client.routes[detail] = DiscoverySourceError('fixture outage') + _, receipt = search_crates('web scraper', 1, client) + self.assertEqual('partial', receipt.status) + self.assertIn(detail, receipt.endpoints) + + def test_huggingface_one_failed_kind_is_partial_not_complete(self): + client = FakeClient() + for kind in ('models', 'datasets', 'spaces'): + client.routes[f'https://huggingface.co/api/{kind}?search=web+scraper&limit=1&full=true'] = [] + client.routes['https://huggingface.co/api/datasets?search=web+scraper&limit=1&full=true'] = DiscoverySourceError('fixture outage') + _, receipt = search_huggingface('web scraper', 3, client) + self.assertEqual('partial', receipt.status) + self.assertIn('fixture outage', receipt.error) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_discovery_providers.py b/tests/test_discovery_providers.py index 5dd21a6..ae15fac 100644 --- a/tests/test_discovery_providers.py +++ b/tests/test_discovery_providers.py @@ -5,7 +5,11 @@ import unittest from pathlib import Path -from brief2ship.discovery_http import DiscoveryHttpClient, HttpPayload +from brief2ship.discovery_http import ( + DiscoveryHttpClient, + DiscoverySourceError, + HttpPayload, +) from brief2ship.discovery_providers import ( canonical_repository_url, enrich_osv, @@ -15,16 +19,21 @@ search_npm, search_pypi, ) +from brief2ship.discovery_query import plan_query class FakeClient(DiscoveryHttpClient): def __init__(self): super().__init__() self.routes: dict[str, object] = {} + self.requested_json: list[str] = [] self.simple = b'safe-web-scraperother' def get_json(self, url, *, headers=None, max_bytes=5_000_000): # noqa: ARG002 + self.requested_json.append(url) value = self.routes[url] + if isinstance(value, DiscoverySourceError): + raise value return value, HttpPayload(200, url, {"x-ratelimit-remaining": "42"}, json.dumps(value).encode()) def post_json(self, url, body, *, max_bytes=5_000_000) -> tuple[object, HttpPayload]: # noqa: ARG002 @@ -81,40 +90,81 @@ def test_github_provider(self): self.assertEqual(receipt.rate_limit_remaining, 42) self.assertEqual(candidates[0].repository_size_kb, 123) - def test_github_retries_zero_result_sentence_with_bounded_focus_query(self): + def test_github_uses_bounded_rank_fusion_even_after_nonzero_results(self): client = FakeClient() - original = ( - "https://api.github.com/search/repositories?" - "q=repo-first+multi-ecosystem+code+discovery+and+reuse+scoring&" - "per_page=1&page=1" - ) - focused = ( + query = "Build a local deterministic web scraper with retry support" + variants = plan_query(query).variants + self.assertEqual(3, len(variants)) + endpoints = [ "https://api.github.com/search/repositories?" - "q=repo+discovery+scoring&per_page=1&page=1" - ) - client.routes[original] = {"items": []} - client.routes[focused] = { - "items": [ - { - "id": 9, - "full_name": "owner/repo-discovery", - "html_url": "https://github.com/owner/repo-discovery", - "description": "repository discovery and scoring", - "license": {"spdx_id": "MIT"}, - "pushed_at": "2026-07-01T00:00:00Z", - } - ] + f"q={variant.replace(' ', '+')}&per_page=2&page=1" + for variant in variants + ] + first = { + "id": 9, + "full_name": "owner/first-only", + "html_url": "https://github.com/owner/first-only", + "description": "first query result", + } + repeated = { + "id": 10, + "full_name": "owner/repeated", + "html_url": "https://github.com/owner/repeated", + "description": "result present in multiple queries", } + client.routes[endpoints[0]] = {"items": [first, repeated]} + client.routes[endpoints[1]] = {"items": [repeated]} + client.routes[endpoints[2]] = {"items": []} - candidates, receipt = search_github( - "repo-first multi-ecosystem code discovery and reuse scoring", - 1, - client, - ) + candidates, receipt = search_github(query, 2, client) + + self.assertEqual(["owner/repeated", "owner/first-only"], [candidate.name for candidate in candidates]) + self.assertEqual([1, 2], [candidate.source_rank for candidate in candidates]) + self.assertEqual(list(variants), receipt.queries) + self.assertEqual(endpoints, receipt.endpoints) + self.assertEqual(endpoints, client.requested_json) + + def test_github_records_partial_query_failures_without_losing_results(self): + client = FakeClient() + query = "Build a local deterministic web scraper with retry support" + variants = plan_query(query).variants + endpoints = [ + "https://api.github.com/search/repositories?" + f"q={variant.replace(' ', '+')}&per_page=1&page=1" + for variant in variants + ] + client.routes[endpoints[0]] = {"items": [{ + "id": 11, + "full_name": "owner/survivor", + "html_url": "https://github.com/owner/survivor", + }]} + client.routes[endpoints[1]] = DiscoverySourceError("fixture timeout") + client.routes[endpoints[2]] = {"items": []} - self.assertEqual(["owner/repo-discovery"], [candidate.name for candidate in candidates]) - self.assertEqual([original, focused], receipt.endpoints) - self.assertTrue(any("focused fallback" in warning for warning in receipt.warnings)) + candidates, receipt = search_github(query, 1, client) + + self.assertEqual(["owner/survivor"], [candidate.name for candidate in candidates]) + self.assertEqual("partial", receipt.status) + self.assertEqual(list(variants), receipt.queries) + self.assertEqual(endpoints, receipt.endpoints) + self.assertTrue(any("fixture timeout" in warning for warning in receipt.warnings)) + + def test_github_all_valid_empty_queries_are_ok_not_failed(self): + client = FakeClient() + query = "Build a local deterministic web scraper with retry support" + variants = plan_query(query).variants + for variant in variants: + endpoint = ( + "https://api.github.com/search/repositories?" + f"q={variant.replace(' ', '+')}&per_page=1&page=1" + ) + client.routes[endpoint] = {"items": []} + + candidates, receipt = search_github(query, 1, client) + + self.assertEqual([], candidates) + self.assertEqual("ok", receipt.status) + self.assertIsNone(receipt.error) def test_pypi_provider_uses_simple_index_and_package_json(self): client = FakeClient() @@ -165,6 +215,68 @@ def test_pypi_name_ties_are_alphabetical_not_reverse_alphabetical(self): self.assertEqual(["safe-alpha"], [candidate.name for candidate in candidates]) + def test_pypi_dependency_metadata_distinguishes_empty_from_unknown(self): + client = FakeClient() + names = ( + "evidence-empty", + "evidence-malformed", + "evidence-missing", + "evidence-null", + ) + client.simple = b"".join( + f'{name}'.encode() for name in names + ) + for name in names: + info: dict[str, object] = { + "name": name, + "version": "1.0.0", + "summary": "dependency evidence", + "package_url": f"https://pypi.org/project/{name}/", + } + if name == "evidence-empty": + info["requires_dist"] = [] + elif name == "evidence-malformed": + info["requires_dist"] = {"unexpected": "mapping"} + elif name == "evidence-null": + info["requires_dist"] = None + client.routes[f"https://pypi.org/pypi/{name}/json"] = { + "info": info, + "releases": {"1.0.0": []}, + } + + with tempfile.TemporaryDirectory() as temporary: + candidates, receipt = search_pypi( + "evidence", + 4, + client, + cache_dir=Path(temporary), + ) + + counts = {candidate.name: candidate.dependency_count for candidate in candidates} + self.assertEqual(0, counts["evidence-empty"]) + self.assertIsNone(counts["evidence-malformed"]) + self.assertIsNone(counts["evidence-missing"]) + self.assertIsNone(counts["evidence-null"]) + self.assertEqual([1, 2, 3, 4], [candidate.source_rank for candidate in candidates]) + self.assertTrue(any("requires_dist" in warning for warning in receipt.warnings)) + + def test_pypi_no_name_matches_is_an_ok_empty_result(self): + client = FakeClient() + client.simple = b'unrelated' + + with tempfile.TemporaryDirectory() as temporary: + candidates, receipt = search_pypi( + "web scraper", + 2, + client, + cache_dir=Path(temporary), + ) + + self.assertEqual([], candidates) + self.assertEqual("ok", receipt.status) + self.assertIsNone(receipt.error) + self.assertEqual(["web scraper"], receipt.queries) + def test_npm_crates_and_huggingface_providers(self): client = FakeClient() npm_search = "https://registry.npmjs.org/-/v1/search?text=web+scraper&size=1" @@ -189,6 +301,79 @@ def test_npm_crates_and_huggingface_providers(self): self.assertEqual(receipt.returned, 3) self.assertEqual({item.license for item in hf}, {"mit"}) + def test_npm_dependency_hydration_keeps_failures_and_malformed_data_unknown(self): + client = FakeClient() + names = ("detail-empty", "detail-failed", "detail-malformed", "version-missing") + search_endpoint = ( + "https://registry.npmjs.org/-/v1/search?text=detail&size=4" + ) + client.routes[search_endpoint] = { + "objects": [ + { + "package": { + "name": name, + "version": "1.0.0", + "links": {"npm": f"https://www.npmjs.com/package/{name}"}, + } + } + for name in names + ] + } + client.routes["https://registry.npmjs.org/detail-empty"] = { + "versions": {"1.0.0": {}} + } + client.routes["https://registry.npmjs.org/detail-failed"] = ( + DiscoverySourceError("fixture detail failure") + ) + client.routes["https://registry.npmjs.org/detail-malformed"] = { + "versions": {"1.0.0": {"dependencies": ["not", "a", "mapping"]}} + } + client.routes["https://registry.npmjs.org/version-missing"] = { + "versions": {"2.0.0": {"dependencies": {}}} + } + + candidates, receipt = search_npm("detail", 4, client) + + counts = {candidate.name: candidate.dependency_count for candidate in candidates} + self.assertEqual(0, counts["detail-empty"]) + self.assertIsNone(counts["detail-failed"]) + self.assertIsNone(counts["detail-malformed"]) + self.assertIsNone(counts["version-missing"]) + self.assertEqual([1, 2, 3, 4], [candidate.source_rank for candidate in candidates]) + self.assertEqual(["detail"], receipt.queries) + self.assertTrue(any("fixture detail failure" in warning for warning in receipt.warnings)) + self.assertTrue(any("dependencies" in warning for warning in receipt.warnings)) + self.assertTrue(any("version 1.0.0" in warning for warning in receipt.warnings)) + + def test_other_registry_providers_use_one_core_query(self): + client = FakeClient() + query = "Build a local web scraper" + core = plan_query(query).core_query + npm_search = ( + "https://registry.npmjs.org/-/v1/search?text=web+scraper&size=1" + ) + client.routes[npm_search] = {"objects": []} + crates_search = ( + "https://crates.io/api/v1/crates?" + "q=web+scraper&sort=relevance&per_page=1&page=1" + ) + client.routes[crates_search] = {"crates": []} + for plural in ("models", "datasets", "spaces"): + endpoint = ( + f"https://huggingface.co/api/{plural}?" + "search=web+scraper&limit=1&full=true" + ) + client.routes[endpoint] = [] + + _, npm_receipt = search_npm(query, 1, client) + _, crates_receipt = search_crates(query, 1, client) + _, hf_receipt = search_huggingface(query, 1, client) + + self.assertEqual("web scraper", core) + self.assertEqual([core], npm_receipt.queries) + self.assertEqual([core], crates_receipt.queries) + self.assertEqual([core], hf_receipt.queries) + def test_osv_findings_are_preserved(self): from brief2ship.discovery_models import Candidate diff --git a/tests/test_discovery_query.py b/tests/test_discovery_query.py new file mode 100644 index 0000000..11934ea --- /dev/null +++ b/tests/test_discovery_query.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import re +import unittest +from dataclasses import FrozenInstanceError, fields + +from brief2ship.discovery_query import QueryPlan, plan_query + + +class QueryPlanTests(unittest.TestCase): + def test_public_contract_is_frozen_and_exact(self): + self.assertEqual( + ["original", "core_query", "constraints", "variants"], + [field.name for field in fields(QueryPlan)], + ) + plan = plan_query("web scraper") + with self.assertRaises(FrozenInstanceError): + plan.core_query = "changed" # type: ignore[misc] + + def test_local_robots_aware_scraper_keeps_domain_as_core_intent(self): + query = "local robots-aware web scraper" + + plan = plan_query(query) + + self.assertEqual(query, plan.original) + self.assertEqual("web scraper", plan.core_query) + self.assertEqual(("local", "robots-aware"), plan.constraints) + self.assertEqual(plan.core_query, plan.variants[0]) + self.assertIn(query, plan.variants) + self.assertLessEqual(len(plan.variants), 3) + + def test_package_names_and_central_function_are_not_dropped(self): + plan = plan_query( + "Build a dependency-free PDF to Markdown converter using PyMuPDF" + ) + manager = plan_query("Build a package manager for Python projects") + + self.assertEqual(("dependency-free",), plan.constraints) + self.assertIn("PDF to Markdown converter", plan.core_query) + self.assertIn("PyMuPDF", plan.core_query) + self.assertIn("package manager", manager.core_query) + + def test_variants_are_deterministic_bounded_and_use_only_original_terms(self): + query = "Please build a local deterministic web scraper with retry support" + + first = plan_query(query) + second = plan_query(query) + + self.assertEqual(first, second) + self.assertLessEqual(len(first.variants), 3) + self.assertEqual(len(first.variants), len(set(first.variants))) + self.assertIn(query, first.variants) + original_terms = set(re.findall(r"[A-Za-z0-9_.+#@/-]+", query.lower())) + for variant in first.variants: + self.assertLessEqual( + set(re.findall(r"[A-Za-z0-9_.+#@/-]+", variant.lower())), + original_terms, + ) + + def test_original_is_retained_exactly_when_whitespace_is_normalized_for_core(self): + query = " Build a web scraper " + + plan = plan_query(query) + + self.assertEqual(query, plan.original) + self.assertEqual("web scraper", plan.core_query) + self.assertIn(query, plan.variants) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_discovery_scoring.py b/tests/test_discovery_scoring.py index 0483aca..fa0f32b 100644 --- a/tests/test_discovery_scoring.py +++ b/tests/test_discovery_scoring.py @@ -4,8 +4,18 @@ from datetime import datetime, timezone from brief2ship.discovery import deduplicate_candidates -from brief2ship.discovery_models import Candidate, DiscoveryConfig -from brief2ship.discovery_scoring import rank_candidates, score_candidate +from brief2ship.discovery_models import ( + Candidate, + DiscoveryConfig, + InspectionResult, + ScoreBreakdown, + TestReceipt, +) +from brief2ship.discovery_scoring import ( + candidate_rank_key, + rank_candidates, + score_candidate, +) class DiscoveryScoringTests(unittest.TestCase): @@ -127,7 +137,8 @@ def test_dedup_prefers_package_identity_and_keeps_github_evidence(self): self.assertEqual(len(merged), 1) self.assertEqual(merged[0].source, "pypi") self.assertEqual(merged[0].stars, 900) - self.assertEqual(merged[0].license, "MIT") + self.assertIsNone(merged[0].license) + self.assertEqual(merged[0].repository_evidence["license"], "MIT") self.assertIn("github:owner/tool", merged[0].aliases) def test_dedup_does_not_mutate_provider_candidates(self): @@ -357,6 +368,174 @@ def test_missing_license_rejects_even_when_total_is_low(self): self.assertEqual("reject", candidate.recommendation) self.assertEqual("blocked", candidate.recommendation_status) + def test_canonical_mit_body_is_recognized_without_approving_freeform_notice(self): + raw_license = '''The MIT License (MIT) + +Copyright (c) 2026 Example Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.''' + candidate = Candidate( + source="github", + name="owner/robots-discovery", + url="https://github.com/owner/robots-discovery", + repository_url="https://github.com/owner/robots-discovery", + description="robots package discovery", + license=raw_license, + vulnerabilities_checked=True, + ) + + score_candidate("robots package discovery", candidate) + + self.assertEqual(raw_license, candidate.license) + self.assertIsNone(candidate.normalized_license) + self.assertEqual("MIT", candidate.license_body_match) + self.assertTrue(candidate.license_review_required) + self.assertTrue(any("license" in blocker for blocker in candidate.hard_blockers)) + + def test_high_health_unrelated_utility_does_not_crowd_relevant_lead(self): + candidates = [ + Candidate( + source="github", + name="authlib/jose", + url="https://github.com/authlib/jose", + repository_url="https://github.com/authlib/jose", + description="A polished general purpose JOSE utility", + license="MIT", + updated_at="2026-07-29T00:00:00Z", + stars=1_000_000, + forks=20_000, + watchers=5_000, + contributors=500, + dependency_count=0, + security_policy=True, + vulnerabilities_checked=True, + test_signals=["tests"], + portability_signals=["cross-platform"], + reuse_signals=["docs", "examples", "package"], + source_rank=1, + ), + Candidate( + source="github", + name="owner/robots-package-discovery", + url="https://github.com/owner/robots-package-discovery", + repository_url="https://github.com/owner/robots-package-discovery", + description="robots-aware package discovery CLI", + license="MIT", + source_rank=20, + ), + ] + + ranked = rank_candidates( + "Build a robots-aware package discovery CLI for Windows that runs locally with no cloud services", + candidates, + now=datetime(2026, 7, 30, tzinfo=timezone.utc), + ) + + self.assertEqual("owner/robots-package-discovery", ranked[0].name) + self.assertIsNotNone(ranked[0].score) + self.assertIsNotNone(ranked[1].score) + assert ranked[0].score is not None + assert ranked[1].score is not None + self.assertGreaterEqual(ranked[0].score.components["feature_match"], 8) + self.assertLess(ranked[1].score.components["feature_match"], 8) + + def test_unknown_evidence_reduces_decision_score_without_inflating_raw_total(self): + candidate = Candidate( + source="github", + name="owner/robots-discovery", + url="https://github.com/owner/robots-discovery", + repository_url="https://github.com/owner/robots-discovery", + description="robots package discovery", + license="MIT", + dependency_count=None, + ) + + score = score_candidate("robots package discovery", candidate) + + self.assertEqual(5.0, score.components["dependency_weight"]) + self.assertEqual(round(max(0.0, score.total - score.unknown_cost), 2), score.decision_score) + self.assertLess(score.decision_score, score.total) + + def test_rank_key_prefers_ready_then_inspected_and_caps_same_source_rank(self): + def tied(name: str, *, status: str, inspected: bool, source_rank: int) -> Candidate: + candidate = Candidate( + source="github", + name=name, + url=f"https://github.com/{name}", + source_rank=source_rank, + recommendation="selective-reuse", + recommendation_status=status, + score=ScoreBreakdown( + total=70.0, + components={"feature_match": 12.0}, + evidence={}, + coverage=0.8, + unknown_cost=4.0, + decision_score=66.0, + ), + ) + if inspected: + candidate.inspection = InspectionResult( + repository_url=candidate.url, + status="inspected", + ) + return candidate + + ready = tied("owner/ready", status="ready", inspected=True, source_rank=500) + inspected = tied("owner/inspected", status="provisional", inspected=True, source_rank=500) + provisional = tied("owner/provisional", status="provisional", inspected=False, source_rank=1) + self.assertLess(candidate_rank_key(ready), candidate_rank_key(inspected)) + self.assertLess(candidate_rank_key(inspected), candidate_rank_key(provisional)) + + capped_a = tied("owner/a", status="provisional", inspected=False, source_rank=21) + capped_b = tied("owner/b", status="provisional", inspected=False, source_rank=10_000) + self.assertEqual(candidate_rank_key(capped_a)[:-2], candidate_rank_key(capped_b)[:-2]) + + def test_requested_constraints_remain_visible_until_verified(self): + candidate = Candidate( + source="github", + name="owner/robots-discovery", + url="https://github.com/owner/robots-discovery", + repository_url="https://github.com/owner/robots-discovery", + description="robots package discovery", + license="MIT", + vulnerabilities_checked=True, + inspection=InspectionResult( + repository_url="https://github.com/owner/robots-discovery", + status="inspected", + test_receipt=TestReceipt(status="passed"), + ), + ) + + score_candidate( + "Build robots package discovery for Windows, local operation, and no cloud services", + candidate, + ) + + combined_checks = "\n".join( + candidate.constraint_checks + candidate.required_checks + ).casefold() + self.assertIn("windows", combined_checks) + self.assertIn("local", combined_checks) + self.assertIn("cloud", combined_checks) + self.assertEqual(len(candidate.required_checks), len(set(candidate.required_checks))) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_discovery_summary.py b/tests/test_discovery_summary.py new file mode 100644 index 0000000..b60573f --- /dev/null +++ b/tests/test_discovery_summary.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import contextlib +import io +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from brief2ship.cli import main +from brief2ship.discovery_models import Candidate, DiscoveryResult, SourceReceipt +from brief2ship.discovery_render import render_discovery_summary + + +class SummaryTests(unittest.TestCase): + def result(self): + visible = Candidate("github", "example/lead", "https://example.invalid/lead") + hidden = Candidate("github", "example/hidden", "https://example.invalid/hidden", + hard_blockers=["license missing", "archived"]) + return DiscoveryResult( + query="web scraper", started_at="fixture", completed_at="fixture", config={}, + candidates=[visible], evaluated_candidates=[visible, hidden], + sources=[SourceReceipt("github", "failed", 2, returned=2, + error="fixture outage", warnings=["fixture warning"])], + overall_recommendation="inconclusive", recommendation_reason="fixture outage", + discovery_status="failed", decision_status="inconclusive", + incomplete_reasons=["github: fixture outage"], + ) + + def test_summary_does_not_hide_failed_sources_or_non_displayed_blockers(self): + result = self.result() + payload = json.loads(render_discovery_summary(result, Path("receipt"))) + self.assertEqual("inconclusive", payload["decision"]) + self.assertEqual("failed", payload["sources"][0]["status"]) + self.assertEqual("fixture outage", payload["sources"][0]["error"]) + self.assertEqual(1, payload["sources"][0]["warning_count"]) + self.assertEqual(2, payload["hard_blocker_count"]) + self.assertEqual(2, payload["evaluated_count"]) + self.assertEqual(1, payload["displayed_count"]) + self.assertIsNone(payload["selected_candidate"]) + self.assertEqual("example/lead", payload["top_candidate"]) + + def test_summary_cli_writes_receipts_then_exits_five_for_inconclusive(self): + with tempfile.TemporaryDirectory() as root, patch("brief2ship.cli.discover_candidates", return_value=self.result()): + stream = io.StringIO() + with contextlib.redirect_stdout(stream): + code = main(["discover", "web scraper", "--summary", "--output", root]) + payload = json.loads(stream.getvalue()) + self.assertEqual(5, code) + for path in payload["receipts"].values(): + self.assertTrue(Path(path).is_file()) + + def test_default_stdout_keeps_single_receipt_path(self): + with tempfile.TemporaryDirectory() as root, patch("brief2ship.cli.discover_candidates", return_value=self.result()): + stream = io.StringIO() + with contextlib.redirect_stdout(stream): + code = main(["discover", "web scraper", "--output", root]) + self.assertEqual(5, code) + self.assertEqual(str(Path(root).resolve() / "discovery.md"), stream.getvalue().strip()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py index dd21303..5aecc74 100644 --- a/tests/test_release_contract.py +++ b/tests/test_release_contract.py @@ -19,7 +19,7 @@ def test_version_is_consistent(self): init = (ROOT / "src/brief2ship/__init__.py").read_text(encoding="utf-8") skill = (ROOT / "skills/brief2ship/SKILL.md").read_text(encoding="utf-8") changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8") - self.assertEqual("0.6.2", version) + self.assertEqual("0.7.0", version) self.assertEqual(version, pyproject["project"]["version"]) self.assertIn(f'__version__ = "{version}"', init) self.assertIn(f"version: {version}", skill) @@ -241,10 +241,35 @@ def test_agent_distribution_metadata_matches_release(self): self.assertEqual(version, plugin["version"]) self.assertEqual(version, marketplace["plugins"][0]["version"]) readme = (ROOT / "README.md").read_text(encoding="utf-8") - self.assertIn(f"brief2ship@v{version}", readme) + changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8") + if f"## {version} - Unreleased" in changelog: + self.assertIn(f"v{version} (unreleased)", readme) + self.assertNotIn(f"brief2ship@v{version}", readme) + else: + self.assertIn(f"brief2ship@v{version}", readme) self.assertIn("npx skills add five0nit/brief2ship --skill brief2ship", readme) self.assertIn("claude plugin install brief2ship@brief2ship", readme) + def test_installed_smoke_uses_interpreter_scripts_scheme(self): + import runpy + from unittest.mock import patch + + script = runpy.run_path(str(ROOT / "scripts/smoke-installed.py")) + with tempfile.TemporaryDirectory() as directory: + for scripts, suffix in (("Python/Scripts", ".exe"), ("venv/Scripts", ".exe"), ("venv/bin", "")): + expected = Path(directory) / scripts / ("brief2ship" + suffix) + with self.subTest(scripts=scripts), patch( + "sysconfig.get_path", return_value=str(expected.parent) + ) as get_path, patch("sysconfig.get_config_var", return_value=suffix): + self.assertEqual(expected, script["_installed_entrypoint"]()) + get_path.assert_called_once_with("scripts") + + def test_ci_exercises_quality_and_both_distributions(self): + workflow = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8") + for required in ("ruff==0.16.6", "pyright==1.1.411", "scripts/benchmark-discovery.py", + "scripts/check-sdist.py", "scripts/smoke-installed.py"): + self.assertIn(required, workflow) + def test_pypi_trusted_publishing_contract(self): workflow = (ROOT / ".github/workflows/publish-pypi.yml").read_text(encoding="utf-8") self.assertIn("environment:\n name: pypi", workflow)