ci: mutation testing with gremlins (just mutation, Test Action job)

.github/workflows/test.yml

@@ -258,6 +258,41 @@ jobs:
           CGO_ENABLED: "1"
           CGO_CFLAGS: "-w"
 
+  # Mutation testing: gremlins changes the code (negated conditions, moved
+  # boundaries, swapped operators) and reruns the package's tests. A mutant the
+  # tests do not notice is a behaviour nothing checks, which line coverage
+  # cannot show. It runs the same command as `just mutation`'s default and
+  # fails when the share of mutants killed drops below the threshold.
+  mutation:
+    name: Mutation testing
+    runs-on: [self-hosted, Linux, X64, vulnetix-local]
+    needs: [lint]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+
+      - name: Set up Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+
+      # gremlins runs `go mod download` itself, so the private modules must be
+      # fetchable before it starts, exactly as for the tests.
+      - name: Configure private fetch of vdb-sca-match and vdb-manifest (not yet in module proxy)
+        run: |
+          go env -w GOPRIVATE=github.com/Vulnetix/vdb-sca-match,github.com/Vulnetix/vdb-manifest
+          git config --global url."https://x-access-token:${{ secrets.PACKAGES_TOKEN }}@github.com/Vulnetix/vdb-sca-match".insteadOf "https://github.com/Vulnetix/vdb-sca-match"
+          git config --global url."https://x-access-token:${{ secrets.PACKAGES_TOKEN }}@github.com/Vulnetix/vdb-manifest".insteadOf "https://github.com/Vulnetix/vdb-manifest"
+
+      # Same command and pins as `just mutation` (CI has no just). The version is
+      # pinned so a gremlins upgrade cannot silently change the result.
+      - name: Run gremlins
+        run: |
+          go run github.com/go-gremlins/gremlins/cmd/gremlins@v0.6.0 unleash \
+            --threshold-efficacy 80 --timeout-coefficient 10 ./internal/purl
+        shell: bash
+
   # Proves the customer-facing signing path end to end, against public Fulcio
   # and public Rekor, using the identity this workflow already has.
   #

justfile

@@ -259,6 +259,15 @@ test-coverage-check:
     go test -v -coverprofile=coverage.out ./...
     go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//' | awk '{if ($1 < 15) exit 1}'
 
+# Mutation testing with gremlins: mutate the package's code and rerun its tests;
+# a mutant that survives is behaviour no test checks. Fails when fewer than
+# `efficacy` percent of the mutants are killed. Pinned, like golangci-lint in CI,
+# so an upgrade cannot change the result unannounced. CI runs the default
+# (.github/workflows/test.yml, job "Mutation testing"); pass another package to
+# survey it, e.g. `just mutation ./internal/versions 0`.
+mutation pkg="./internal/purl" efficacy="80":
+    go run github.com/go-gremlins/gremlins/cmd/gremlins@v0.6.0 unleash --threshold-efficacy {{efficacy}} --timeout-coefficient 10 {{pkg}}
+
 # Comprehensive test suite
 test-all: test test-coverage-check