From 5cb417e09f16b5bdeb28ab15726afed8a7916850 Mon Sep 17 00:00:00 2001
From: Christopher Langton <chris@vulnetix.com>
Date: Thu, 12 Nov 2026 22:06:46 +1100
Subject: [PATCH] ci: mutation testing with gremlins (just mutation, Test
 Action job)

Adds a pinned gremlins v0.6.0 run as a justfile recipe (any package,
default internal/purl at 80% efficacy) and as a "Mutation testing" job
in the Test Action workflow. The job sets up the private module fetch
first, since gremlins runs go mod download itself. The threshold sits
under the measured baseline (84.62% on internal/purl today; 100% once
the #225 tests land), so a test change that stops catching a mutant
fails the job.

Closes #224
---
 .github/workflows/test.yml | 35 +++++++++++++++++++++++++++++++++++
 justfile                   |  9 +++++++++
 2 files changed, 44 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5b30f12b49..ba67b0db74 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -258,6 +258,41 @@ jobs:
           CGO_ENABLED: "1"
           CGO_CFLAGS: "-w"
 
+  # Mutation testing: gremlins changes the code (negated conditions, moved
+  # boundaries, swapped operators) and reruns the package's tests. A mutant the
+  # tests do not notice is a behaviour nothing checks, which line coverage
+  # cannot show. It runs the same command as `just mutation`'s default and
+  # fails when the share of mutants killed drops below the threshold.
+  mutation:
+    name: Mutation testing
+    runs-on: [self-hosted, Linux, X64, vulnetix-local]
+    needs: [lint]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+
+      - name: Set up Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+
+      # gremlins runs `go mod download` itself, so the private modules must be
+      # fetchable before it starts, exactly as for the tests.
+      - name: Configure private fetch of vdb-sca-match and vdb-manifest (not yet in module proxy)
+        run: |
+          go env -w GOPRIVATE=github.com/Vulnetix/vdb-sca-match,github.com/Vulnetix/vdb-manifest
+          git config --global url."https://x-access-token:${{ secrets.PACKAGES_TOKEN }}@github.com/Vulnetix/vdb-sca-match".insteadOf "https://github.com/Vulnetix/vdb-sca-match"
+          git config --global url."https://x-access-token:${{ secrets.PACKAGES_TOKEN }}@github.com/Vulnetix/vdb-manifest".insteadOf "https://github.com/Vulnetix/vdb-manifest"
+
+      # Same command and pins as `just mutation` (CI has no just). The version is
+      # pinned so a gremlins upgrade cannot silently change the result.
+      - name: Run gremlins
+        run: |
+          go run github.com/go-gremlins/gremlins/cmd/gremlins@v0.6.0 unleash \
+            --threshold-efficacy 80 --timeout-coefficient 10 ./internal/purl
+        shell: bash
+
   # Proves the customer-facing signing path end to end, against public Fulcio
   # and public Rekor, using the identity this workflow already has.
   #
diff --git a/justfile b/justfile
index f7b5e360aa..5e9b971226 100644
--- a/justfile
+++ b/justfile
@@ -259,6 +259,15 @@ test-coverage-check:
     go test -v -coverprofile=coverage.out ./...
     go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//' | awk '{if ($1 < 15) exit 1}'
 
+# Mutation testing with gremlins: mutate the package's code and rerun its tests;
+# a mutant that survives is behaviour no test checks. Fails when fewer than
+# `efficacy` percent of the mutants are killed. Pinned, like golangci-lint in CI,
+# so an upgrade cannot change the result unannounced. CI runs the default
+# (.github/workflows/test.yml, job "Mutation testing"); pass another package to
+# survey it, e.g. `just mutation ./internal/versions 0`.
+mutation pkg="./internal/purl" efficacy="80":
+    go run github.com/go-gremlins/gremlins/cmd/gremlins@v0.6.0 unleash --threshold-efficacy {{efficacy}} --timeout-coefficient 10 {{pkg}}
+
 # Comprehensive test suite
 test-all: test test-coverage-check