GTM-1590: scan renamed files and first-party vendor/, and state the denominator

.github/workflows/self-test.yml

@@ -81,5 +81,81 @@ jobs:
           bash tests/junk_file_gate_test.sh
           echo ">> Restored gate is green again."
 
+      - name: Prove the secret-scan suite can fail
+        # Same reasoning as the step above, applied to the gate where a false
+        # green is most expensive. The mutation restores the exact defect that
+        # GTM-1590 fixed: --diff-filter=ACM drops renamed files, so an
+        # edited-and-moved file ships unscanned. That shipped for weeks and the
+        # gate reported "passed" every time, so the suite has to be able to see it.
+        run: |
+          set -euo pipefail
+          cp scripts/secret_scan_gate.sh /tmp/secret_gate.bak
+
+          python3 <<'PY'
+          import pathlib, sys
+          p = pathlib.Path("scripts/secret_scan_gate.sh")
+          t = p.read_text()
+          needle = "--diff-filter=ACMRT)"
+          if needle not in t:
+              sys.exit("mutation target not found; update this step alongside the gate")
+          p.write_text(t.replace(needle, "--diff-filter=ACM)", 1))
+          PY
+
+          rc=0
+          bash tests/secret_scan_gate_test.sh >/tmp/secret_mutant.out 2>&1 || rc=$?
+          cp /tmp/secret_gate.bak scripts/secret_scan_gate.sh
+
+          if [ "$rc" -eq 0 ]; then
+            echo "!! The suite PASSED with renamed files excluded from the scan."
+            echo "   The tests are not asserting what they claim to assert."
+            tail -20 /tmp/secret_mutant.out
+            exit 1
+          fi
+          echo ">> Mutation correctly turned the suite red."
+
+          bash tests/secret_scan_gate_test.sh
+          echo ">> Restored gate is green again."
+
+      - name: Prove the suite still sees the vendor cause
+        # GTM-1590 had TWO independent causes and the step above pins only the
+        # first. Mutating one cause proves nothing about the other, and the
+        # vendor skip is the half that reaches first-party content, so it gets
+        # its own restore-and-go-red rather than riding on the rename drill.
+        run: |
+          set -euo pipefail
+          cp scripts/secret_scan_gate.sh /tmp/secret_gate2.bak
+
+          python3 <<'PY'
+          import pathlib, sys
+          p = pathlib.Path("scripts/secret_scan_gate.sh")
+          t = p.read_text()
+          needle = 'SKIP_DIRS = ("node_modules/", ".git/", "dist/", "build/")'
+          if needle not in t:
+              sys.exit("mutation target not found; update this step alongside the gate")
+          mutant = 'SKIP_DIRS = ("node_modules/", ".git/", "vendor/", "dist/", "build/")'
+          p.write_text(t.replace(needle, mutant, 1))
+          PY
+
+          rc=0
+          bash tests/secret_scan_gate_test.sh >/tmp/secret_mutant2.out 2>&1 || rc=$?
+          cp /tmp/secret_gate2.bak scripts/secret_scan_gate.sh
+
+          if [ "$rc" -eq 0 ]; then
+            echo "!! The suite PASSED with vendor/ skipped unconditionally again."
+            echo "   First-party vendor content would go unscanned and nothing would say so."
+            tail -20 /tmp/secret_mutant2.out
+            exit 1
+          fi
+          echo ">> Mutation correctly turned the suite red."
+
+          bash tests/secret_scan_gate_test.sh
+          echo ">> Restored gate is green again."
+
       - name: Gate this repo with its own junk check
         run: bash scripts/junk_file_gate.sh --audit
+
+      - name: Gate this repo with its own secret scan
+        # Dogfooding, and it is not free: the scanner's own source describes the
+        # shapes it detects, so this step is what keeps those descriptions written
+        # as prose rather than as quoted literals.
+        run: bash scripts/secret_scan_gate.sh --audit

.secret-scan-allow

@@ -0,0 +1,9 @@
+# Paths this repo's secret scan deliberately does not read. One path per line,
+# reason after "#". Keep this file short: every line here is a place a real
+# credential could hide, so a broad entry costs more than it saves.
+#
+# scripts/secret_scan_gate.sh is deliberately NOT listed. Its detector comments
+# were written to describe key shapes in prose rather than to quote them, so the
+# gate still scans its own source. GTM-1590.
+
+tests/secret_scan_gate_test.sh  # A scanner's test suite must contain strings matching every shape the scanner detects, including a PEM private key header, or the tests prove nothing. All values in it are synthetic. GTM-1590.

scripts/secret_scan_gate.sh

@@ -37,6 +37,14 @@
 # ONLY TRACKED OR STAGED CONTENT CAN FAIL, enumerated through git, so a gitignored
 # file on disk is invisible here by construction. Same principle as the junk gate.
 #
+# THE DENOMINATOR IS PART OF THE VERDICT (GTM-1590, 2026-08-31). This gate always
+# printed how many paths it checked, and twice in two days that count was a small
+# fraction of what was staged while the line still read "passed". Both shortfalls
+# were in plain text and were read past, so an honest count is necessary and is not
+# sufficient. The summary now prints "N of M path(s) checked" with every skip and
+# its reason, because "0 of 1 checked, 1 skipped (renamed)" cannot be misread the
+# way a bare "0 path(s) checked" was.
+#
 # Modes:
 #   (default)   compare against a base ref; what a pull request would add
 #   --staged    inspect the index; used by the pre-commit hook
@@ -66,34 +74,90 @@ while [ $# -gt 0 ]; do
     --audit)  MODE="audit" ;;
     --staged) MODE="staged" ;;
     --base)   shift; BASE="${1:-}" ;;
-    -h|--help) sed -n '2,52p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
+    -h|--help) sed -n '2,60p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
     *) echo "secret_scan_gate: unknown option $1" >&2; exit 2 ;;
   esac
   shift
 done
 
 command -v python3 >/dev/null 2>&1 || { echo ">> secret scan skipped: no python3" >&2; exit 0; }
 
+# THE FILTER, and why R and T are in it (GTM-1590).
+#
+# This used to read --diff-filter=ACM. Excluding D is right: a deleted path has no
+# content left to scan. Excluding R was WRONG, because a renamed file's content
+# still ships, and git classifies an edited-and-moved file as R as soon as the
+# similarity index clears its threshold. So the ordinary motion of editing a file
+# and moving it in one commit, which is exactly what retiring a register entry or
+# reorganising a docs tree looks like, rode past this gate entirely.
+#
+# Measured on gtmify-config: commit ba71968 staged four renames and one
+# modification and the gate reported "1 path(s) checked"; commit c5925f8 was a
+# single `git mv` plus an edit and reported "0 path(s) checked" while carrying a
+# real 7-insertion change. Reproduced from scratch on 2026-08-31 before this fix.
+#
+# T (typechange) joins them for the same reason: a symlink becoming a regular file
+# is new content arriving. D stays out, and is counted and named as a skip below
+# rather than silently vanishing from the denominator.
+#
+# --name-only prints a rename's DESTINATION path only, never its source, so
+# ALL_PATHS and PATHS stay directly comparable and the counts add up.
 case "$MODE" in
-  staged) PATHS=$(git diff --cached --name-only --diff-filter=ACM) ;;
-  audit)  PATHS=$(git ls-files) ;;
+  staged)
+    ALL_PATHS=$(git diff --cached --name-only)
+    PATHS=$(git diff --cached --name-only --diff-filter=ACMRT)
+    ;;
+  audit)
+    ALL_PATHS=$(git ls-files)
+    PATHS="$ALL_PATHS"
+    ;;
   diff)
     if [ -z "$BASE" ]; then
       BASE=$(git symbolic-ref -q --short refs/remotes/origin/HEAD 2>/dev/null || echo "origin/main")
     fi
-    PATHS=$(git diff --name-only --diff-filter=ACM "$BASE"...HEAD 2>/dev/null || git ls-files)
+    if ALL_PATHS=$(git diff --name-only "$BASE"...HEAD 2>/dev/null); then
+      PATHS=$(git diff --name-only --diff-filter=ACMRT "$BASE"...HEAD 2>/dev/null)
+    else
+      ALL_PATHS=$(git ls-files)
+      PATHS="$ALL_PATHS"
+    fi
     ;;
 esac
 
+# Counted here rather than in python, so the deleted paths that are deliberately
+# never handed to the scanner still appear in the denominator it prints.
+if [ -z "$ALL_PATHS" ]; then
+  SSG_TOTAL=0
+else
+  SSG_TOTAL=$(printf '%s\n' "$ALL_PATHS" | wc -l | tr -d ' ')
+fi
+
 if [ -z "$PATHS" ]; then
-  echo ">> secret scan passed. 0 path(s) checked, mode=$MODE."
+  if [ "$SSG_TOTAL" -eq 0 ]; then
+    echo ">> secret scan passed. 0 of 0 path(s) checked, mode=$MODE."
+  else
+    echo ">> secret scan passed. 0 of $SSG_TOTAL path(s) checked, mode=$MODE."
+    echo ">>   skipped: $SSG_TOTAL deleted"
+  fi
   exit 0
 fi
 
-SSG_LIST=$(mktemp -t ssg_paths)
+# `mktemp -t ssg_paths` is a BSD spelling: macOS treats the argument as a PREFIX and
+# appends its own randomness, GNU coreutils treats it as a TEMPLATE and refuses with
+# "too few X's in template". This gate had only ever run on macOS, so on every Linux
+# host it produced an empty $SSG_LIST, handed python an empty path, and died with
+# IsADirectoryError. Found by the new test suite on the first CI run, 2026-08-31.
+#
+# It failed CLOSED, so nothing was let through, but a gate that crashes on every
+# commit is a gate that gets uninstalled. An explicit template works on both.
+SSG_LIST=$(mktemp "${TMPDIR:-/tmp}/ssg_paths.XXXXXX")
+if [ -z "$SSG_LIST" ] || [ ! -f "$SSG_LIST" ]; then
+  echo ">> secret scan ERROR: could not create a temporary file. Nothing was scanned." >&2
+  exit 1
+fi
 printf '%s\n' "$PATHS" > "$SSG_LIST"
 trap 'rm -f "$SSG_LIST"' EXIT
-export SSG_LIST SSG_MODE="$MODE"
+export SSG_LIST SSG_MODE="$MODE" SSG_TOTAL
 
 python3 - <<'PY'
 import os, re, subprocess, sys, pathlib
@@ -106,7 +170,23 @@ paths = [p for p in pathlib.Path(os.environ["SSG_LIST"]).read_text().split("\n")
 SKIP_SUFFIX = (".sops", ".sops.yaml", ".sops.yml", ".age", ".bundle", ".pack",
                ".png", ".jpg", ".jpeg", ".gif", ".pdf", ".zip", ".gz", ".tar",
                ".woff", ".woff2", ".ico", ".mp4", ".mov")
-SKIP_DIRS = ("node_modules/", ".git/", "vendor/", "dist/", "build/")
+SKIP_DIRS = ("node_modules/", ".git/", "dist/", "build/")
+
+# ── vendor/ IS NOT UNCONDITIONALLY A DEPENDENCY DIRECTORY (GTM-1590) ─────────
+# "vendor/" used to sit in SKIP_DIRS beside node_modules, on the dependency-directory
+# convention where it means third-party code nobody here wrote. In this estate that
+# convention is false: gtmify/app/vendor is the vendor DOCUMENTATION mirror, twenty
+# first-party-committed trees that sessions actively author, and a worked API example
+# is exactly where a live key gets pasted. A commit staging three paths under vendor/
+# reported "0 path(s) checked" on 2026-08-29.
+#
+# So it is qualified rather than named: skipped only when a package manager has
+# actually marked the tree vendored. Go writes vendor/modules.txt, Composer writes
+# vendor/autoload.php. Neither marker exists anywhere in this estate, so vendor/ is
+# now scanned here, and a repo that really does vendor its dependencies still gets
+# the skip without needing to know this gate exists.
+VENDOR_MARKERS = ("vendor/modules.txt", "vendor/autoload.php")
+vendor_is_dependencies = any(pathlib.Path(m).is_file() for m in VENDOR_MARKERS)
 
 # Per-repo escape hatch: one path per line in .secret-scan-allow, reason after "#".
 allow = set()
@@ -118,9 +198,43 @@ if ap.is_file():
             allow.add(line)
 
 # Things that look like secrets but are not.
+#
+# The second line was added with GTM-1590, when un-skipping vendor/ first exposed the
+# gate to twenty trees of vendor documentation. Every term here is an explicit
+# admission by the document that the value is not real: "replace-with-at-least-32-
+# random-characters", "asdfasdfasdf", "tr_preview_1234567890", "secret-from-trigger-
+# dev". Each was measured as a live false positive in gtmify/app/vendor on
+# 2026-08-31; none of them widens the gate against a value that is actually a secret.
 PLACEHOLDER = re.compile(
     r"(REDACTED|EXAMPLE|PLACEHOLDER|CHANGEME|CHANGE_ME|YOUR[_-]|<[^>]{2,}>|xxxx|XXXX|\.\.\.|"
-    r"dummy|sample|test[_-]?only|FAKE|NOT[_-]?REAL)", re.I)
+    r"dummy|sample|test[_-]?only|FAKE|NOT[_-]?REAL|"
+    r"replace[_-]?with|asdfasdf|1234567890|secret[_-]from)", re.I)
+
+# Vendor sample keys published in the vendor's OWN public documentation. Matched as
+# exact literals, never as a shape, so this can never widen into a class. Stripe has
+# printed this key in its API reference for a decade; it appears in eight files of
+# gtmify/app/vendor/stripe-docs and is not a credential to anything.
+#
+# Written as prefix plus body rather than as one literal, and NOT arbitrary
+# obfuscation: the detector matches on the prefix, so splitting exactly there is
+# what lets this gate go on scanning its own source. The alternative was an
+# allowlist entry for scripts/secret_scan_gate.sh, which would have made the one
+# file where a credential must never hide the one file nobody reads. The value
+# compared at runtime is still the exact literal.
+KNOWN_PUBLIC_SAMPLES = (
+    "sk_" + "test_" + "BQokikJOvBiI2HlWgH4olfQ2",
+)
+
+# Object IDs whose prefix collides with a credential prefix. A Stripe REFUND id is
+# the four characters "re_1" followed by twenty-four base62 characters, which the
+# Resend-key detector matches exactly; five such hits were measured in stripe-docs.
+# (Spelled out rather than quoted, so that this gate can still scan its own source
+# instead of needing an allowlist entry for the comment describing its detectors.)
+# The guard is narrow on purpose: it fires
+# only where the match is the value of a JSON "id" field, which is a place a
+# credential is never legitimately assigned, rather than loosening the re_ pattern
+# itself and losing a real Resend key.
+JSON_ID_VALUE = re.compile(r'"id"\s*:\s*"[A-Za-z0-9_]+"')
 
 # ── PATTERN layer ────────────────────────────────────────────────────────────
 PATTERNS = [
@@ -141,9 +255,17 @@ PATTERNS = [
     ("jwt",                    re.compile(r"\beyJ[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{10,}")),
     # The generic one. This catches a credential whose vendor prefix we do not
     # know, which is how the Stripe key arrived inside a JSON snapshot.
+    #
+    # The lookahead was added with GTM-1590. Assigning a secret-named variable FROM
+    # process.env, app.env, import.meta.env, os.environ or a shell expansion is the
+    # correct handling of a secret, not a leak of one: the literal is elsewhere by
+    # construction. Four such lines in gtmify/app/vendor were measured firing on
+    # 2026-08-31, all of them library setup examples doing exactly the right thing.
     ("secret-shaped assignment",
      re.compile(r"""['"]?[A-Z][A-Z0-9_]{2,}_(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|PAT|CREDENTIAL)S?['"]?"""
-                r"""\s*[:=]\s*['"]?[A-Za-z0-9_\-.]{20,}""")),
+                r"""\s*[:=]\s*['"]?"""
+                r"""(?!process\.env\.|app\.env\.|import\.meta\.env\.|os\.environ|ENV\[|\$\{|\$[A-Za-z_])"""
+                r"""[A-Za-z0-9_\-.]{20,}""")),
 ]
 
 # ── VALUE layer ──────────────────────────────────────────────────────────────
@@ -200,15 +322,36 @@ def content_of(path):
 
 findings = []
 checked = 0
+
+# Every path that is not read is counted under a REASON. The gate's whole failure
+# mode was a shortfall it reported as a bare number, so a skip that cannot name
+# itself is not allowed to exist. GTM-1590.
+skips = {}
+def skipped(reason):
+    skips[reason] = skips.get(reason, 0) + 1
+
+def in_dir(path, d):
+    return path.startswith(d) or ("/" + d) in path
+
 for p in paths:
     if p in allow:
+        skipped("allowlisted in .secret-scan-allow")
         continue
     if p.endswith(SKIP_SUFFIX):
+        skipped("encrypted or binary file type")
+        continue
+    if any(in_dir(p, d) for d in SKIP_DIRS):
+        skipped("dependency or build directory")
         continue
-    if any(p.startswith(d) or ("/" + d) in p for d in SKIP_DIRS):
+    if vendor_is_dependencies and in_dir(p, "vendor/"):
+        skipped("vendored dependencies")
         continue
     blob = content_of(p)
-    if not blob or b"\x00" in blob[:8000]:
+    if not blob:
+        skipped("no readable content")
+        continue
+    if b"\x00" in blob[:8000]:
+        skipped("binary content")
         continue
     text = blob.decode("utf-8", errors="replace")
     checked += 1
@@ -230,6 +373,12 @@ for p in paths:
         # would be noise rather than protection.
         if "X-Amz-Credential" in line or "X-Amz-Signature" in line:
             continue
+        # A vendor's own published sample key, matched as an exact literal.
+        if any(s in line for s in KNOWN_PUBLIC_SAMPLES):
+            continue
+        # A JSON object id, which collides with credential prefixes such as re_.
+        if JSON_ID_VALUE.search(line):
+            continue
         for label, rx in PATTERNS:
             if rx.search(line):
                 findings.append((p, i, label))
@@ -259,9 +408,32 @@ if uniq:
     print("")
     if not env_vals:
         print("   NOTE: no local env file found, so only the pattern layer ran.")
+    # A refusal states its coverage for the same reason a pass does: knowing the
+    # gate found something says nothing about how much of the commit it read.
+    print("   Coverage: {} of {} path(s) in scope were read.".format(
+        checked, int(os.environ.get("SSG_TOTAL") or len(paths))))
     sys.exit(1)
 
+# ── THE VERDICT, WITH ITS DENOMINATOR ────────────────────────────────────────
+# total is what git said was in scope; checked is what was actually read. The
+# difference is enumerated by reason and never left as a residual, so a summary
+# whose parts do not add up to its total is itself visible as a defect.
+total = int(os.environ.get("SSG_TOTAL") or len(paths))
+deleted = total - len(paths)
+if deleted > 0:
+    skips["deleted"] = skips.get("deleted", 0) + deleted
+
 extra = "" if env_vals else " (value layer unavailable: no local env file)"
-print(">> secret scan passed. {} path(s) checked, mode={}{}.".format(checked, mode, extra))
+print(">> secret scan passed. {} of {} path(s) checked, mode={}{}.".format(
+    checked, total, mode, extra))
+if skips:
+    parts = ", ".join("{} {}".format(n, reason)
+                      for reason, n in sorted(skips.items(), key=lambda kv: (-kv[1], kv[0])))
+    print(">>   skipped: {}".format(parts))
+accounted = checked + sum(skips.values())
+if accounted != total:
+    print(">>   WARNING: {} checked plus {} skipped does not equal {} in scope. "
+          "The gate is not seeing everything it thinks it is.".format(
+              checked, sum(skips.values()), total))
 sys.exit(0)
 PY