diff --git a/.github/workflows/lint-include.yml b/.github/workflows/lint-include.yml index 7a949b40..55122bff 100644 --- a/.github/workflows/lint-include.yml +++ b/.github/workflows/lint-include.yml @@ -151,6 +151,16 @@ on: required: false type: boolean default: false + lint-line-endings: + description: "Report text files committed with CRLF line endings (scripts/lint/line-endings.sh). See lint.yml." + required: false + type: boolean + default: false + lint-line-endings-blocking: + description: "Report CRLF line endings as errors and fail the line-endings linter; false reports warnings only" + required: false + type: boolean + default: true upload-sarif: description: "Upload checkov/trivy/zizmor/ruff/biome/psscriptanalyzer results to GitHub code scanning (requires GitHub Advanced Security). Set false for private repos without GHAS to gate on exit codes with console output instead." @@ -209,5 +219,7 @@ jobs: lint-config-drift: ${{ inputs.lint-config-drift }} lint-inline-scripts: ${{ inputs.lint-inline-scripts }} lint-inline-scripts-blocking: ${{ inputs.lint-inline-scripts-blocking }} + lint-line-endings: ${{ inputs.lint-line-endings }} + lint-line-endings-blocking: ${{ inputs.lint-line-endings-blocking }} upload-sarif: ${{ inputs.upload-sarif }} lint-job-mode: ${{ inputs.lint-job-mode }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5b48d736..f3bccf0f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -146,6 +146,19 @@ on: required: false type: boolean default: false + lint-line-endings: + description: >- + Report text files committed with CRLF or mixed line endings (scripts/lint/line-endings.sh, git ls-files --eol). + A path that needs CRLF gets its own .gitattributes rule: `eol=crlf` (stored with LF) or `-text` (stored byte + for byte). + required: false + type: boolean + default: true + lint-line-endings-blocking: + description: "Report CRLF line endings as errors and fail the line-endings linter; false reports warnings only" + required: false + type: boolean + default: true upload-sarif: description: "Upload checkov/trivy/zizmor/ruff/biome/psscriptanalyzer results to GitHub code scanning (requires GitHub Advanced Security). Set false for private repos without GHAS to gate on exit codes with console output instead." @@ -962,6 +975,43 @@ jobs: # The runner's system Python ships PyYAML; a caller's mise Python may not. run: /usr/bin/python3 "${RUNNER_TEMP}/devsecninja-lint/scripts/lint-inline-scripts.py" + line-endings: + name: line endings + if: ${{ inputs.lint-line-endings && (inputs.lint-job-mode == 'parallel' || (inputs.lint-job-mode == 'auto' && !github.event.repository.private)) + }} + continue-on-error: ${{ ! inputs.lint-fail-on-error }} + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Checkout lint scripts + # The linter scripts ship with this reusable workflow: fetch them from + # the same commit of DevSecNinja/.github that the caller pinned. + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} + path: .devsecninja-lint + sparse-checkout: |- + scripts/lint/ + scripts/lint-inline-scripts.py + sparse-checkout-cone-mode: false + persist-credentials: false + + - name: Move lint scripts out of the workspace + # Outside the workspace the caller's linters don't scan them, and the + # code checkout doesn't delete them. + run: mv .devsecninja-lint "${RUNNER_TEMP}/devsecninja-lint" + + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Check line endings + env: + LINT_LINE_ENDINGS_BLOCKING: ${{ inputs.lint-line-endings-blocking }} + run: bash "${RUNNER_TEMP}/devsecninja-lint/scripts/lint/line-endings.sh" + all-linters: # Single-job mode (`lint-job-mode: single`, or `auto` on a private # repository). Every enabled linter runs as a step of this one job, so a @@ -1013,6 +1063,16 @@ jobs: fetch-depth: ${{ !inputs.lint-gitleaks && 1 || 0 }} persist-credentials: false + - name: Check line endings + # Needs only git: runs before the tool setup, so a failed mise + # install doesn't skip it. + id: line-endings + if: ${{ inputs.lint-line-endings }} + continue-on-error: true + env: + LINT_LINE_ENDINGS_BLOCKING: ${{ inputs.lint-line-endings-blocking }} + run: bash "${RUNNER_TEMP}/devsecninja-lint/scripts/lint/line-endings.sh" + - name: Find Python files # Repositories without Python skip ruff (and its mise install). id: python-files diff --git a/config-sync/templates/.gitattributes b/config-sync/templates/.gitattributes index 80505c21..f38dfaa7 100644 --- a/config-sync/templates/.gitattributes +++ b/config-sync/templates/.gitattributes @@ -9,6 +9,8 @@ # CRLF gets its own rule below, for example: # *.ps1 text eol=crlf CRLF in the checkout, LF in the repository # *.ics -text stored byte for byte (RFC 5545 calendars need CRLF) +# The shared Lint workflow (lint-line-endings) fails on text files committed +# with CRLF, except -text paths. * text=auto eol=lf # Scripts stay LF even when a repository changes the default above. diff --git a/config-sync/templates/README.md b/config-sync/templates/README.md index 43ac9acd..7e7cf702 100644 --- a/config-sync/templates/README.md +++ b/config-sync/templates/README.md @@ -74,7 +74,10 @@ uses another default lists `.gitattributes` in `.github/.config-sync-ignore`, which also skips that warning. Adding the file doesn't convert files already committed with CRLF; run -`git add --renormalize .` and commit the result. +`git add .gitattributes` and `git add --renormalize .`, and commit both. The +shared Lint workflow's `lint-line-endings` check fails on text files committed +with CRLF or mixed line endings, except `-text` paths (see +[docs/architecture.md](../../docs/architecture.md#lint-lintyml)). ## renovate.json5 diff --git a/docs/architecture.md b/docs/architecture.md index 6aae2130..7fa68abe 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -254,6 +254,19 @@ runs it blocking. A later major release makes blocking the default. The checker runs with the runner's system Python, which ships PyYAML (`ubuntu-24.04`). +The `line endings` linter (`scripts/lint/line-endings.sh`) reports every text +file committed with CRLF or mixed line endings. It reads `git ls-files --eol`, +so git's own `.gitattributes` resolution decides: a path with `-text` (or +`binary`) is stored byte for byte and skipped, and an `eol=crlf` path passes +once git stores it with LF (it only checks it out with CRLF); one still stored +with CRLF is reported like any other. Those rules are +the only way to keep CRLF; the linter has no exclude input. Findings are errors +that fail the linter (and the workflow only with `lint-fail-on-error: true`); +`lint-line-endings-blocking: false` reports them as warnings. To fix them, put +`* text=auto eol=lf` in `.gitattributes` (config-sync bootstraps it, see +[Templates](#templates-config-synctemplates)), run `git add .gitattributes` and +`git add --renormalize .`, and commit both. The linter needs only git. + **zizmor config.** After v4.7.0, zizmor runs with the shared [`scripts/lint/zizmor.yml`](../scripts/lint/zizmor.yml), which ships with `lint.yml` at the commit the caller pinned, unless the repository has its own @@ -293,6 +306,8 @@ its own commit (`lint-include.yml` calling `lint.yml`), `./` in callers. | `lint-config-drift` | Config-drift check. Default: `false`. | | `lint-inline-scripts` | Inline-script check (see above). Default: `true`. After v4.4.0. | | `lint-inline-scripts-blocking` | Report inline scripts as errors. Default: `false`. After v4.4.0. | +| `lint-line-endings` | CRLF line-endings check (see above). Default: `true`. After v4.7.0. | +| `lint-line-endings-blocking` | Report CRLF as errors and fail. Default: `true`. After v4.7.0. | **Example caller:** diff --git a/scripts/lint/line-endings.sh b/scripts/lint/line-endings.sh new file mode 100644 index 00000000..e63a499f --- /dev/null +++ b/scripts/lint/line-endings.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# +# Fails when a tracked text file is committed with CRLF (or mixed) line +# endings, using git's own attribute resolution (`git ls-files --eol`): +# a path whose .gitattributes rule unsets `text` (`-text`, or `binary`) is +# stored byte for byte and skipped. An `eol=crlf` path passes once git +# stores it with LF (it only checks it out with CRLF); one still stored +# with CRLF is reported, like any other, until `git add --renormalize`. +# A step of .github/workflows/lint.yml, run the same way in both lint modes. +# Env: LINT_LINE_ENDINGS_BLOCKING (true: errors, fail; false: warnings only). + +set -euo pipefail +# Bytes, not characters: in a UTF-8 locale `read -d ''` can swallow the NUL +# after a path that ends in an incomplete multibyte sequence. +export LC_ALL=C + +blocking="${LINT_LINE_ENDINGS_BLOCKING:-true}" +case "${blocking}" in +true) level=error ;; +false) level=warning ;; +*) + echo "::error::lint-line-endings-blocking must be true or false (got '${blocking}')." + exit 2 + ;; +esac + +# Escapes the message of a workflow command, and (escape_property) a +# property value such as file=.... +escape_data() { + local value="$1" + value="${value//%/%25}" + value="${value//$'\r'/%0D}" + value="${value//$'\n'/%0A}" + printf '%s' "${value}" +} +escape_property() { + local value + value="$(escape_data "$1")" + value="${value//:/%3A}" + value="${value//,/%2C}" + printf '%s' "${value}" +} + +max_annotations=50 +checked=0 +found=() +# A file, not a process substitution: `set -e` then stops on a failing +# `git ls-files` (no repository, dubious ownership) instead of passing. +listing="$(mktemp)" +trap 'rm -f "${listing}"' EXIT +git ls-files --eol -z >"${listing}" +# Each record is "i/ w/ attr/\t"; +# split at the first tab only, so a path keeps its own tabs. +while IFS= read -r -d '' record; do + checked=$((checked + 1)) + info="${record%%$'\t'*}" + path="${record#*$'\t'}" + case "${info%% *}" in + i/crlf | i/mixed) ;; + *) continue ;; + esac + attributes="${info#*attr/}" + case " ${attributes} " in + *" -text "*) continue ;; + *) ;; + esac + found+=("${path}") +done <"${listing}" + +if [ "${#found[@]}" -eq 0 ]; then + echo "✓ No text file is committed with CRLF line endings (${checked} files checked)." + exit 0 +fi + +for path in "${found[@]:0:${max_annotations}}"; do + echo "::${level} file=$(escape_property "${path}"),title=CRLF line endings::$(escape_data "${path}") is committed with CRLF line endings." +done +[ "${#found[@]}" -le "${max_annotations}" ] || + echo "… and $((${#found[@]} - max_annotations)) more file(s)." +echo "::${level} title=CRLF line endings::${#found[@]} text file(s) are committed with CRLF line endings. Store them with LF: put \"* text=auto eol=lf\" in .gitattributes, run \"git add .gitattributes\" and \"git add --renormalize .\", and commit both. A file that needs CRLF gets its own .gitattributes rule: \"eol=crlf\" (CRLF in the checkout, LF in the repository) or \"-text\" (stored byte for byte)." +[ "${blocking}" = false ] || exit 1 diff --git a/tests/lint-include.py b/tests/lint-include.py index 71de8551..35b2d4c9 100755 --- a/tests/lint-include.py +++ b/tests/lint-include.py @@ -35,7 +35,12 @@ LINT_REF = "$/.github/workflows/lint.yml" # Boolean `lint-*` inputs of lint.yml that are options rather than linters. -NOT_LINTERS = {"lint-fail-on-error", "lint-inline-scripts-blocking", "lint-biome-with-eslint"} +NOT_LINTERS = { + "lint-fail-on-error", + "lint-inline-scripts-blocking", + "lint-biome-with-eslint", + "lint-line-endings-blocking", +} LEVELS = {"none": 0, "read": 1, "write": 2} diff --git a/tests/lint-job-modes-contract.sh b/tests/lint-job-modes-contract.sh index ef32cb98..9c7ba233 100755 --- a/tests/lint-job-modes-contract.sh +++ b/tests/lint-job-modes-contract.sh @@ -14,7 +14,7 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" workflow="${repo_root}/.github/workflows/lint.yml" combined_job="all-linters" -linter_jobs=(dprint yamlfmt yamllint actionlint gitleaks go shellcheck shfmt checkov trivy zizmor ruff biome psscriptanalyzer config-drift inline-scripts) +linter_jobs=(dprint yamlfmt yamllint actionlint gitleaks go shellcheck shfmt checkov trivy zizmor ruff biome psscriptanalyzer config-drift inline-scripts line-endings) parallel_guard="(inputs.lint-job-mode == 'parallel' || (inputs.lint-job-mode == 'auto' && !github.event.repository.private))" fail() { diff --git a/tests/lint-scripts.sh b/tests/lint-scripts.sh index 94717d39..24098cb6 100755 --- a/tests/lint-scripts.sh +++ b/tests/lint-scripts.sh @@ -482,4 +482,57 @@ expect_status 1 "drifted config" expect_out "::error::.yamllint.yaml has drifted from central config" "drifted config" expect_out "+central" "drifted config" +# line-endings fails on text files committed with CRLF or mixed line endings. +# -text and binary paths are stored byte for byte and skipped; eol=crlf paths +# pass, because git stores them with LF. +eol="${tmp_dir}/eol" +mkdir -p "${eol}/sub" +git -C "${eol}" init -q +printf 'a\nb\n' >"${eol}/lf.txt" +printf 'a\r\nb\r\n' >"${eol}/feed.ics" +printf 'a\r\n' >"${eol}/asset.dat" +printf 'a\0\r\n' >"${eol}/blob" +printf 'a\r\n' >"${eol}/run.bat" +printf '* text=auto eol=lf\n*.ics -text\n*.dat binary\n*.bat text eol=crlf\n' >"${eol}/.gitattributes" +git -C "${eol}" -c core.autocrlf=false add -A +run_in "${eol}" line-endings +expect_status 0 "LF, -text, binary and eol=crlf files" +expect_out "✓ No text file is committed with CRLF line endings (6 files checked)." "LF, -text, binary and eol=crlf files" + +# Files committed with CRLF before the .gitattributes rule, staged as is. +stage_raw() { + local sha + printf '%b' "$2" >"${eol}/$1" + sha="$(git -C "${eol}" hash-object -w --no-filters -- "$1")" + git -C "${eol}" update-index --add --cacheinfo "100644,${sha},$1" +} +stage_raw sub/crlf.txt 'a\r\nb\r\n' +stage_raw mixed.md 'a\nb\r\n' +stage_raw 'odd,name%.txt' 'a\r\n' +# An eol=crlf path still stored with CRLF (committed before its rule) is reported too. +stage_raw legacy.bat 'a\r\n' +# A path with leading and trailing tabs (index only: not every file system allows them). +tab_sha="$(printf 'a\r\n' | git -C "${eol}" hash-object -w --no-filters --stdin)" +git -C "${eol}" -c core.protectNTFS=false update-index --add --cacheinfo "100644,${tab_sha},"$'\ttabs.txt\t' +run_in "${eol}" line-endings +expect_status 1 "CRLF files" +expect_out "::error file=sub/crlf.txt,title=CRLF line endings::sub/crlf.txt is committed with CRLF line endings." "CRLF files" +expect_out "::error file=mixed.md,title=CRLF line endings::" "mixed line endings" +expect_out "::error file=odd%2Cname%25.txt,title=CRLF line endings::odd,name%25.txt is committed" "an escaped path" +expect_out "::error file="$'\ttabs.txt\t'",title=CRLF line endings::" "a path with tabs" +expect_out "::error file=legacy.bat,title=CRLF line endings::" "an eol=crlf path stored with CRLF" +expect_out "5 text file(s) are committed with CRLF line endings" "CRLF files" +expect_out 'run "git add .gitattributes" and "git add --renormalize ."' "the fix" +! grep -Fq "feed.ics" <<<"${out}" || fail "a -text path must not be reported: ${out}" +run_in "${eol}" line-endings LINT_LINE_ENDINGS_BLOCKING=false +expect_status 0 "CRLF files without blocking" +expect_out "::warning file=sub/crlf.txt,title=CRLF line endings::" "CRLF files without blocking" +run_in "${eol}" line-endings LINT_LINE_ENDINGS_BLOCKING=bogus +expect_status 2 "an invalid blocking value" +# A failing scan fails the linter instead of reporting no CRLF. +mkdir -p "${tmp_dir}/not-a-repo" +run_in "${tmp_dir}/not-a-repo" line-endings GIT_CEILING_DIRECTORIES="${tmp_dir}" +expect_failure "outside a git repository" +! grep -Fq "No text file is committed" <<<"${out}" || fail "a failed scan must not report success: ${out}" + echo "lint scripts: OK" diff --git a/workflow-templates/lint.yml b/workflow-templates/lint.yml index a03a2df2..3b7e90f8 100644 --- a/workflow-templates/lint.yml +++ b/workflow-templates/lint.yml @@ -82,3 +82,7 @@ jobs: # after v4.4.0; bump the pin above before uncommenting. # lint-inline-scripts: true # lint-inline-scripts-blocking: false # true: errors instead of warnings + # Optional: report text files committed with CRLF line endings. Needs a + # lint.yml release after v4.7.0; bump the pin above before uncommenting. + # lint-line-endings: true + # lint-line-endings-blocking: true # false: warnings instead of errors