fix(license): bound GitHub licence lookups so scan jobs stop draining the repository's API quota

.github/workflows/vulnetix.yml

@@ -259,6 +259,7 @@ jobs:
           org-id: ${{ secrets.VULNETIX_ORG_ID }}
           api-key: ${{ secrets.VULNETIX_API_KEY }}
           fail-on-findings: false
+          github-token: ""
       - uses: Vulnetix/cli/github-advisories@v4.24.30
         with:
           org-id: ${{ secrets.VULNETIX_ORG_ID }}
@@ -277,6 +278,7 @@ jobs:
           org-id: ${{ secrets.VULNETIX_ORG_ID }}
           api-key: ${{ secrets.VULNETIX_API_KEY }}
           fail-on-findings: false
+          github-token: ""
       - uses: Vulnetix/cli/bom-validate@v4.24.30
         with:
           org-id: ${{ secrets.VULNETIX_ORG_ID }}
@@ -391,6 +393,7 @@ jobs:
           org-id: ${{ secrets.VULNETIX_ORG_ID }}
           api-key: ${{ secrets.VULNETIX_API_KEY }}
           fail-on-findings: false
+          github-token: ""
       - uses: Vulnetix/cli/upload@v4.24.30
         with:
           org-id: ${{ secrets.VULNETIX_ORG_ID }}

AGENTS.md

@@ -384,6 +384,29 @@ than HTTP 404 still marks the repo indefinite, so the same misses are stored.
 `VULNETIX_GH_PARITY=1 go test ./internal/license -run TestGHAPIDirectMatchesGH`
 compares the two against the live API; run it when adding an expression.
 
+Those lookups spend the GitHub API quota that, inside GitHub Actions, the whole
+repository shares. On the push that merged #341, the scan jobs' lookups spent
+4,913 requests in under three and a half minutes, and SCA's SARIF upload and
+analyze's forge pass found the quota gone. So:
+
+- **Validate the name first.** `validOwnerRepo` drops a name GitHub cannot
+  hold before any request: `$/sbom` from `uses: $/sbom`, a glob, `<your`.
+- **Keep the chain short.** It runs repository, then `/license`, then the root
+  listing and the files it names, and a 404 on the repository ends it. There is
+  no second pass with `GITHUB_TOKEN`: gh already sends it.
+- **Spend through one budget.** Every request goes through `githubGet` or
+  `ghAPICall`, which spend from `internal/ghquota`, the budget analyze's forge
+  pass uses. It is seeded from `/rate_limit`, which is free.
+- **Keep a larger share in Actions.** There the lookups keep 80% of the limit
+  (`ciLicenseShare`), above the forge pass's half.
+- **Stop on a rate-limit answer.** What they did not check stays `UNKNOWN`,
+  and `reportGitHubLookups` says how many repositories went unchecked. In
+  Actions it also logs each batch's request count.
+
+`TestLookupChainCost` and `TestLookupsKeepTheirShareInGitHubActions` pin this
+behaviour. `vulnetix gha init` gives no token to a scanner run as an extra's
+`prior` step (`priorScanners`), because such a step publishes nothing.
+
 ### SBOM/CDX Subcommand
 
 The `cdx` command (alias `sbom`, implemented in `cmd/cdx.go`) writes one offline CycloneDX document — no VDB lookup, upload, memory write, quality gate or container daemon. Discovery passes: manifests/lockfiles, installed package trees (`internal/ecosystems`), container package DBs (dpkg/apk/pacman), Dockerfile/compose/k8s/Helm, CI/CD pipeline files, shell scripts + Makefile/justfile recipes, and **compiled artefacts** via `internal/binpkg`. AIBOM and CBOM components are merged into the same file.

cmd/gha_init.go

@@ -44,6 +44,11 @@ get a sast/<language> job each, and the largest job carries a rest step so the
 jobs together cover what the high-level sast action covers; three or more use the
 high-level action. Package managers work the same way with sca/<manager>.
 
+The job token's GitHub API quota belongs to the whole repository, so the
+workflow spends it only where it publishes. Scanners get the token off pull
+requests only. A scanner run as an extra's prior step gets none, because it
+only writes the file the extra reads.
+
 Refs are written as @v4. Run gh actions-lock afterwards to narrow them to exact
 tags and lock them; --check ignores refs, so a locked workflow stays current.
 

cmd/license.go

@@ -218,7 +218,15 @@
 Manifests are discovered by walking the directory tree, parsed locally,
 and each package's license is resolved from manifest fields or an embedded
 SPDX license database. When authenticated, license findings are submitted to
-Vulnetix so the scan is recorded; unauthenticated scans stay entirely local.
+Vulnetix so the scan is recorded; unauthenticated scans submit nothing.
+
+A license those cannot name is looked up in the Vulnetix store (when
+authenticated), the package's registry and deps.dev, and then, with a GitHub
+token (GITHUB_TOKEN, GH_TOKEN or the gh CLI's), on GitHub. The GitHub lookups
+stop while part of the token's hourly rate limit is left: half, or 80% inside
+GitHub Actions, where the job token's quota belongs to the whole repository.
+They also stop when GitHub answers with a rate limit. A license they did not
+check stays UNKNOWN, and a warning says how many repositories went unchecked.
 
 License findings are stored in the CycloneDX SBOM at .vulnetix/sbom.cdx.json
 alongside vulnerability findings (neither overwrites the other).
@@ -252,7 +260,7 @@
 	RunE: runLicense,
 }
 
 func runLicense(cmd *cobra.Command, args []string) (retErr error) {
 	dctx := display.FromCommand(cmd)
 	rootPath, _ := cmd.Flags().GetString("path")
 	maxDepth, _ := cmd.Flags().GetInt("depth")

docs/github-actions.md

@@ -75,6 +75,27 @@ report declares what it did not fetch. The workflow `vulnetix gha init` writes p
 run (`workflow_dispatch`) reads the full `--window-days` window from GitHub. The git history
 window stays at 365 days in every case.
 
+The scanners that resolve licences (`sca`, `license`, `sbom`, `containers` and `scan`) ask
+GitHub for a repository's licence when the Vulnetix store, the package registry and deps.dev
+cannot name it, with the same job token. Those lookups spend from the same quota:
+
+- **A name GitHub cannot hold is never asked about.** That includes a workflow's `uses: $/sbom`,
+  a glob and a template placeholder.
+- **The lookup chain is short.** The repository answers first. A 404 there ends the chain;
+  otherwise the next steps are GitHub's own licence detection, then the licence files that
+  the root listing names, so a repository costs one to four requests.
+- **Inside GitHub Actions the lookups keep 80% of the token's hourly limit for everything
+  else.** They stop at 4,000 left of 5,000, above the half that analyze's forge pass keeps.
+  Every job of a run sees the same remaining count, so the floor holds for the whole run.
+- **They also stop as soon as GitHub answers with a rate limit.** A licence they did not
+  check stays `UNKNOWN`, and a warning names how many repositories went unchecked.
+- **Each job logs what it spent.** In Actions, every batch of lookups prints a line such as
+  `GitHub licence lookups: 31 requests for 26 repositories (4880 of 5000 left)`.
+
+`vulnetix gha init` passes the token to a scanner only off pull requests. It passes no token
+to a scanner that runs as an extra's `prior` step, because that step only writes the file the
+extra reads.
+
 They are generated, not hand-written: `tools/actiongen/spec.go` is the table, and
 `go generate ./tools/actiongen` writes every `<name>/action.yml`, every
 `docs/github-actions/<name>.md` and `action-lib/commands.sh`. `TestGeneratedFilesAreCurrent`

internal/analyze/forge/budget.go

@@ -1,134 +1,14 @@
 package forge
 
-// The request budget.
-//
-// GitHub's rate limit is the single most common way a tool like this produces a wrong
-// answer. There are three ways to get it wrong and all three are in the prior art:
-//
-//   - issue-metrics catches the rate-limit exception and calls sys.exit(1). The whole run is
-//     lost, including the work already done.
-//   - Several tools sleep until the reset, which for a core-quota exhaustion is up to an
-//     hour. A CI job does not have an hour.
-//   - The worst option, and the tempting one: catch the error, return what you have, and say
-//     nothing. The report then quietly understates every count it touched, and looks exactly
-//     like a report from a repository that simply has less going on.
-//
-// So: the budget stops the collector *before* the quota runs out, and what was not fetched
-// is declared. A metric that ran out of budget is `truncated` with an `omittedCount` and a
-// reason. It is not a failure, and it is not a silence.
+// The request budget lives in internal/ghquota, which the licence lookups share: both
+// spend the same token, and inside GitHub Actions the same repository-wide quota.
 
-import (
-	"errors"
-	"fmt"
-	"sync"
+import "github.com/vulnetix/cli/v4/internal/ghquota"
 
-	"github.com/google/go-github/v66/github"
-)
+type budget = ghquota.Budget
 
-// reserve is the least number of requests we refuse to spend. Leaving headroom matters
-// because this token is usually shared — with other steps in the same CI job, and in
-// vdb-manager's case with other processors — and draining it to zero breaks them, not us.
-const reserve = 100
+// newBudget keeps half the token's hourly limit for the other users of the token.
+func newBudget() *budget { return ghquota.NewBudget() }
 
-// reserveShare is the fraction of the hourly limit we refuse to spend, when it is more than
-// reserve. A fixed 100 was not enough headroom: inside GitHub Actions the job token's quota
-// (1,000 an hour) belongs to the whole repository, and a forge pass on every push to the
-// default branch spent it down to 100. CodeQL's SARIF upload and the release dispatch then
-// failed with "API rate limit exceeded for installation". Keeping half the limit bounds
-// what one run can take, whatever else runs beside it.
-const reserveShare = 0.5
-
-// floor is the remaining quota at which the collector stops, for a token with this hourly
-// limit.
-func floor(limit int) int {
-	return max(reserve, int(float64(limit)*reserveShare))
-}
-
-type budget struct {
-	mu sync.Mutex
-
-	remaining int
-	limit     int
-	known     bool
-
-	// spent counts requests we made. Used when GitHub does not tell us the remaining quota
-	// (enterprise instances sometimes do not) so we can still bound the work.
-	spent int
-
-	exhausted bool
-}
-
-func newBudget() *budget {
-	return &budget{}
-}
-
-// Observe records the rate-limit state GitHub reported on a response. Every call the
-// collector makes should pass its response through here.
-func (b *budget) Observe(resp *github.Response) {
-	if resp == nil {
-		return
-	}
-	b.mu.Lock()
-	defer b.mu.Unlock()
-
-	b.spent++
-	if resp.Rate.Limit > 0 {
-		b.remaining = resp.Rate.Remaining
-		b.limit = resp.Rate.Limit
-		b.known = true
-		if b.remaining <= floor(b.limit) {
-			b.exhausted = true
-		}
-	}
-}
-
-// Exhausted reports whether the collector should stop. A caller that ignores this and keeps
-// going will get a 403 and lose the run; a caller that respects it keeps everything it has
-// already gathered.
-func (b *budget) Exhausted() bool {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-
-	return b.exhausted
-}
-
-// Remaining reports the quota left, and whether GitHub actually told us.
-func (b *budget) Remaining() (n int, known bool) {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-
-	return b.remaining, b.known
-}
-
-// Spent reports how many requests were made.
-func (b *budget) Spent() int {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-
-	return b.spent
-}
-
-// Reason explains, in the words that will end up in the report's truncationReason, why the
-// collector stopped early.
-func (b *budget) Reason() string {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-
-	if !b.exhausted {
-		return ""
-	}
-
-	return fmt.Sprintf("Stopped at this run's share of the GitHub API rate limit (%d of %d requests left, "+
-		"kept for other jobs that share the token); the remaining items were not fetched. "+
-		"Re-run when the quota resets, or narrow --forge-window-days.", b.remaining, b.limit)
-}
-
-// IsRateLimit reports whether an error is GitHub telling us to stop. Distinguishing it from
-// a real failure matters: one means "come back later", the other means something is broken,
-// and treating them the same makes both harder to diagnose.
-func IsRateLimit(err error) bool {
-	var rl *github.RateLimitError
-	var abuse *github.AbuseRateLimitError
-
-	return errors.As(err, &rl) || errors.As(err, &abuse)
-}
+// IsRateLimit reports whether an error is GitHub telling us to stop.
+func IsRateLimit(err error) bool { return ghquota.IsRateLimit(err) }

internal/ghainit/render.go

@@ -304,6 +304,17 @@ func actionRef(action, ref string) string {
 	return "Vulnetix/cli/" + action + "@" + ref
 }
 
+// priorScanners are the scanners a prior step can run. A prior step only writes
+// the file its extra reads and publishes nothing, so the one thing the job token
+// bought it was licence lookups against the repository's GitHub API quota; the
+// action's own default token is withheld from it on every event. Two such steps
+// (sbom before bom-validate and before upload) looked up the same licences as
+// the sbom job on every push.
+var priorScanners = map[string]bool{
+	"scan": true, "sca": true, "sast": true, "secrets": true, "iac": true, "containers": true,
+	"license": true, "sbom": true, "cbom": true, "aibom": true,
+}
+
 func (r *renderer) extraJob(e Extra) job {
 	id := e.ID
 	if id == "" {
@@ -315,6 +326,9 @@ func (r *renderer) extraJob(e Extra) job {
 		if a != "setup" { // setup takes none of these
 			p.with = [][2]string{{"org-id", "${{ secrets.VULNETIX_ORG_ID }}"}, {"api-key", "${{ secrets.VULNETIX_API_KEY }}"},
 				{"fail-on-findings", "false"}}
+			if priorScanners[strings.SplitN(a, "/", 2)[0]] {
+				p.with = append(p.with, [2]string{"github-token", ""})
+			}
 		}
 		j.steps = append(j.steps, p)
 	}

internal/ghainit/render_test.go

@@ -164,6 +164,50 @@ func TestScannersGetNoTokenOnPullRequests(t *testing.T) {
 	}
 }
 
+// A prior step writes the file its extra reads and publishes nothing, so a scanner run
+// there gets no token, on any event: with the action's default token it looked up
+// licences against the repository's GitHub API quota.
+func TestPriorScannerStepsGetNoToken(t *testing.T) {
+	p := DefaultPolicy()
+	p.Extras = []Extra{
+		{Action: "bom-validate", Prior: []string{"sbom"}},
+		{Action: "github-advisories", Prior: []string{"sast/go"}},
+		{Action: "report", Prior: []string{"gha-status"}},
+	}
+	out := Render(p, &Signals{}, Options{})
+	var wf struct {
+		Jobs map[string]struct {
+			Steps []struct {
+				Uses string             `yaml:"uses"`
+				With map[string]*string `yaml:"with"`
+			} `yaml:"steps"`
+		} `yaml:"jobs"`
+	}
+	if err := yaml.Unmarshal([]byte(out), &wf); err != nil {
+		t.Fatalf("rendered workflow is not YAML: %v\n%s", err, out)
+	}
+	for id, prior := range map[string]string{"bom-validate": "sbom", "github-advisories": "sast/go", "report": "gha-status"} {
+		steps := wf.Jobs[id].Steps
+		i := 0
+		for i < len(steps) && !strings.HasPrefix(steps[i].Uses, "Vulnetix/cli/") {
+			i++
+		}
+		if i >= len(steps)-1 || !strings.HasPrefix(steps[i].Uses, "Vulnetix/cli/"+prior+"@") {
+			t.Fatalf("%s: prior step missing:\n%s", id, out)
+		}
+		tok, set := steps[i].With["github-token"]
+		if prior == "gha-status" {
+			if set {
+				t.Errorf("%s: a prior that is not a scanner keeps the action's default token", id)
+			}
+			continue
+		}
+		if !set || tok == nil || *tok != "" {
+			t.Errorf("%s: prior %s github-token = %v, want an empty string\n%s", id, prior, tok, out)
+		}
+	}
+}
+
 func TestSetupExtraTakesNoInputs(t *testing.T) {
 	p := DefaultPolicy()
 	p.Extras = []Extra{{Action: "setup", Then: []string{"vulnetix version"}}}