From 9b548b8512bfd2a42eaa3f5ca7d846fff0dfa85c Mon Sep 17 00:00:00 2001 From: Jean-Paul van Ravensberg <14926452+DevSecNinja@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:26:21 +0100 Subject: [PATCH 1/4] feat(config-sync): bootstrap a .gitattributes with LF line endings Refs #578 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/config-sync.yml | 6 +- config-sync/templates/.gitattributes | 83 ++++++++++++++++++++++++++++ config-sync/templates/README.md | 33 ++++++++++- docs/architecture.md | 4 +- scripts/config-sync/sync-files.sh | 22 +++++++- tests/config-sync-renovate-stub.sh | 1 + tests/config-sync-scripts.sh | 72 +++++++++++++++++++++++- 7 files changed, 211 insertions(+), 10 deletions(-) create mode 100644 config-sync/templates/.gitattributes diff --git a/.github/workflows/config-sync.yml b/.github/workflows/config-sync.yml index ef069c27..0c712da2 100644 --- a/.github/workflows/config-sync.yml +++ b/.github/workflows/config-sync.yml @@ -13,7 +13,9 @@ name: Reusable — Config Sync # template is always bootstrapped: it is a stub that extends the central # `.renovate/default.json5` preset and holds the repo's own Renovate # rules, so it is never overwritten. The sync warns when an existing -# `renovate.json5` does not extend that preset. +# `renovate.json5` does not extend that preset. The `.gitattributes` +# template (LF line endings) is always bootstrapped as well, and the +# sync warns when an existing `.gitattributes` has no LF default. # # Per-repo opt-out: list paths (one per line, relative to repo root) # in `.github/.config-sync-ignore` to skip them during sync. Blank @@ -52,7 +54,7 @@ on: inputs: sync-templates: description: "Also sync template files (only copies if file does not exist locally — will not overwrite customized - files). renovate.json5 is bootstrapped regardless." + files). renovate.json5 and .gitattributes are bootstrapped regardless." required: false type: boolean default: false diff --git a/config-sync/templates/.gitattributes b/config-sync/templates/.gitattributes new file mode 100644 index 00000000..80505c21 --- /dev/null +++ b/config-sync/templates/.gitattributes @@ -0,0 +1,83 @@ +# Line endings for this repository, bootstrapped by DevSecNinja/.github +# config-sync (config-sync/templates/.gitattributes). Config-sync adds it only +# when the repository has no .gitattributes and never overwrites it: the +# repository owns this file. Add repository-specific rules at the end; later +# lines win. Based on the shared ~/.gitattributes in DevSecNinja/dotfiles +# (home/dot_gitattributes). + +# LF in the repository and in every checkout, on every OS. A path that needs +# CRLF gets its own rule below, for example: +# *.ps1 text eol=crlf CRLF in the checkout, LF in the repository +# *.ics -text stored byte for byte (RFC 5545 calendars need CRLF) +* text=auto eol=lf + +# Scripts stay LF even when a repository changes the default above. +*.bash text eol=lf +*.fish text eol=lf +*.ksh text eol=lf +*.sh text eol=lf +*.zsh text eol=lf + +# cmd.exe misreads labels and GOTO in batch files with LF line endings. +*.bat text eol=crlf +*.cmd text eol=crlf + +# A patch keeps the line endings of the files it changes. +*.patch -text + +# SVG is XML: text, not binary. +*.svg text + +# Binary files: no line-ending conversion and no text diff. +# Images +*.avif binary +*.eps binary +*.gif binary +*.ico binary +*.jpeg binary +*.jpg binary +*.png binary +*.tif binary +*.tiff binary +*.webp binary +# Documents +*.docx binary +*.pdf binary +*.pptx binary +*.xlsx binary +# Fonts +*.eot binary +*.otf binary +*.ttf binary +*.woff binary +*.woff2 binary +# Audio and video +*.mp3 binary +*.mp4 binary +*.wav binary +*.webm binary +# Archives +*.7z binary +*.bz binary +*.bz2 binary +*.bzip2 binary +*.gz binary +*.lz binary +*.lzma binary +*.rar binary +*.tar binary +*.taz binary +*.tbz binary +*.tbz2 binary +*.tgz binary +*.tlz binary +*.txz binary +*.xz binary +*.Z binary +*.zip binary +*.zst binary +# Executables and libraries +*.dll binary +*.dylib binary +*.exe binary +*.so binary diff --git a/config-sync/templates/README.md b/config-sync/templates/README.md index 11be0897..997ebb4f 100644 --- a/config-sync/templates/README.md +++ b/config-sync/templates/README.md @@ -44,8 +44,37 @@ ai-toolkit; see These files are **starting points** for repos that need them. They are only copied when `sync-templates: true` is set on the caller, and only if the target path does not already exist — local edits are -never overwritten. `renovate.json5` is the exception to the first rule: -it is bootstrapped even without `sync-templates`. +never overwritten. `renovate.json5` and `.gitattributes` are the +exceptions to the first rule: they are bootstrapped even without +`sync-templates`. + +## .gitattributes + +LF line endings for every text file, in the repository and in every +checkout, on every OS: `* text=auto eol=lf`, plus `binary` for common +binary types, LF for shell scripts and CRLF checkouts for `*.bat`/`*.cmd` +(cmd.exe misreads labels in LF batch files). It starts from the shared +`~/.gitattributes` in +[DevSecNinja/dotfiles](https://github.com/DevSecNinja/dotfiles/blob/main/home/dot_gitattributes). + +Config-sync creates it when the repo has no `.gitattributes` and never +overwrites it, so the repo owns the file. Give a path that needs CRLF its own +rule at the end, where later lines win: + +```gitattributes +# CRLF in the checkout, LF in the repository (e.g. Authenticode-signed scripts). +*.ps1 text eol=crlf +# Stored byte for byte: iCalendar (RFC 5545) requires CRLF. +*.ics -text +``` + +Each sync run warns when an existing `.gitattributes` has no LF default (the +last `*` line that sets `eol` must set `eol=lf`). A repo that deliberately +uses another default lists `.gitattributes` in `.github/.config-sync-ignore`, +which also skips that warning. + +Adding the file doesn't convert files already committed with CRLF; run +`git add --renormalize .` and commit the result. ## renovate.json5 diff --git a/docs/architecture.md b/docs/architecture.md index 81544490..0807ff6a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1131,7 +1131,9 @@ These are **starting-point** files. Config-sync copies them only when the caller sets `sync-templates: true` and the file does not exist yet; it never overwrites them. `renovate.json5` is always bootstrapped: it is a stub that extends [`.renovate/default.json5`](#consuming-the-preset) and holds the -repository's own Renovate rules. See +repository's own Renovate rules. `.gitattributes` is always bootstrapped too: +it makes LF the line ending of every text file, and each sync warns when an +existing `.gitattributes` has no LF default. See [`config-sync/templates/README.md`](../config-sync/templates/README.md) for details. diff --git a/scripts/config-sync/sync-files.sh b/scripts/config-sync/sync-files.sh index 636dd781..5117cac8 100644 --- a/scripts/config-sync/sync-files.sh +++ b/scripts/config-sync/sync-files.sh @@ -187,11 +187,27 @@ if [ "${SYNC_TEMPLATES}" = "true" ]; then echo "=== Syncing template files (new files only) ===" sync_tree templates else - # The Renovate stub carries the org defaults, so every repo - # gets one even without sync-templates. + # The Renovate stub carries the org defaults and .gitattributes the + # LF line endings, so every repo gets both even without + # sync-templates. echo "" - echo "=== Bootstrapping renovate.json5 (new file only) ===" + echo "=== Bootstrapping renovate.json5 and .gitattributes (new files only) ===" sync_tree templates renovate.json5 + sync_tree templates .gitattributes +fi + +# .gitattributes is repo-owned as well; only check that LF stays the +# default line ending: the last `*` line that sets `eol` sets `eol=lf`. +# A repository that needs another default lists .gitattributes in +# its ignore file, which also skips this check. Exit 0: LF default. +lf_default() { + awk '{ sub(/\r$/, "") } + $1 == "*" { for (i = 2; i <= NF; i++) if ($i ~ /^eol=/) eol = substr($i, 5) } + END { exit eol != "lf" }' "$1" +} +# shellcheck disable=SC2310 # is_ignored and lf_default only test; their status is the answer. +if [ -f .gitattributes ] && ! is_ignored .gitattributes && ! lf_default .gitattributes; then + echo "::warning file=.gitattributes,title=.gitattributes has no LF default::Add \"* text=auto eol=lf\" above the repo-specific rules, so every text file is stored and checked out with LF. Give a path that needs CRLF its own rule (\"eol=crlf\" or \"-text\"). See https://github.com/DevSecNinja/.github/blob/main/config-sync/templates/README.md#gitattributes" fi # renovate.json5 is repo-owned; only check that its top-level diff --git a/tests/config-sync-renovate-stub.sh b/tests/config-sync-renovate-stub.sh index fa8e8404..f51da991 100644 --- a/tests/config-sync-renovate-stub.sh +++ b/tests/config-sync-renovate-stub.sh @@ -81,6 +81,7 @@ run_sync() { echo "central" >"${source}/config-sync/files/.editorconfig" # Already in sync, so renovate.json5 is the only file that can change. echo "central" >"${consumer}/.editorconfig" + cp "${repo_root}/config-sync/templates/.gitattributes" "${consumer}/.gitattributes" git -C "${source}" init --quiet git -C "${source}" -c user.name=t -c user.email=t@t \ -c commit.gpgsign=false commit --quiet --allow-empty -m init diff --git a/tests/config-sync-scripts.sh b/tests/config-sync-scripts.sh index 1a7913e5..34fcceb9 100644 --- a/tests/config-sync-scripts.sh +++ b/tests/config-sync-scripts.sh @@ -2,8 +2,8 @@ # # Tests scripts/config-sync/ensure-label.sh, which # .github/workflows/config-sync.yml runs before it opens the sync PR (#537), -# and the templates that scripts/config-sync/sync-files.sh skips. The rest of -# sync-files.sh is covered by +# the templates that scripts/config-sync/sync-files.sh skips, and its +# .gitattributes bootstrap and LF check. The rest of sync-files.sh is covered by # tests/config-sync-renovate-stub.sh and tests/config-sync-ai-toolkit-instructions.sh. set -euo pipefail @@ -87,3 +87,71 @@ grep -Fq "biome.json skipped: configured in biome.jsonc" "${tmp_dir}/log" || fail "the skipped biome.json must be logged: $(cat "${tmp_dir}/log")" echo "✓ config-sync: sync-files.sh keeps the ruff.toml and biome.json templates out of repositories that configure them elsewhere." + +# sync-files.sh bootstraps .gitattributes even without sync-templates, never +# overwrites it, and warns when an existing one has no LF default. +gitattributes_template="${repo_root}/config-sync/templates/.gitattributes" + +# Runs the sync without sync-templates in a fresh consumer checkout. $1 is the +# content of its .gitattributes (empty: no file, printed with %b), $2 of its +# .config-sync-ignore. +sync_bootstrap() { + rm -rf "${consumer}" "${tmp_dir}/output" + mkdir -p "${consumer}/.config-sync-source/config-sync" "${consumer}/.github" + cp -R "${repo_root}/config-sync/templates" "${consumer}/.config-sync-source/config-sync/templates" + git -C "${consumer}/.config-sync-source" init --quiet + git -C "${consumer}/.config-sync-source" -c user.name=t -c user.email=t@t \ + -c commit.gpgsign=false commit --quiet --allow-empty -m init + # Already present, so .gitattributes is the only file that can change. + printf '{ extends: ["github>DevSecNinja/.github//.renovate/default.json5"] }\n' >"${consumer}/renovate.json5" + [ -z "$1" ] || printf '%b' "$1" >"${consumer}/.gitattributes" + [ -z "$2" ] || printf '%s\n' "$2" >"${consumer}/.github/.config-sync-ignore" + ( + cd "${consumer}" + GITHUB_REPOSITORY=DevSecNinja/example GITHUB_OUTPUT="${tmp_dir}/output" SYNC_TEMPLATES=false \ + IGNORE_FILE=.github/.config-sync-ignore SOURCE_DIR=.config-sync-source/config-sync \ + bash "${sync_script}" >"${tmp_dir}/log" 2>&1 + ) || fail "sync step failed: $(cat "${tmp_dir}/log")" +} + +lf_warned() { grep -Fq '::warning file=.gitattributes,' "${tmp_dir}/log"; } + +sync_bootstrap "" "" +cmp -s "${gitattributes_template}" "${consumer}/.gitattributes" || + fail "a repository without .gitattributes must get the template, even without sync-templates" +grep -Fxq updated=1 "${tmp_dir}/output" || fail "bootstrapping .gitattributes must open a sync PR" +! lf_warned || fail "the .gitattributes template must not trigger the LF warning: $(cat "${tmp_dir}/log")" +[ ! -e "${consumer}/.mise.toml" ] || fail "other templates still require sync-templates" + +own='* text=auto eol=lf\n*.ics -text\n' +sync_bootstrap "${own}" "" +[ "$(cat "${consumer}/.gitattributes")" = "$(printf '%b' "${own}")" ] || fail "an existing .gitattributes must never be overwritten" +grep -Fxq updated=0 "${tmp_dir}/output" || fail "an existing .gitattributes must not open a sync PR" +! lf_warned || fail "a .gitattributes with an LF default must not warn" + +lf_cases=( + '* text=auto eol=lf\r\n*.ps1 text eol=crlf\r\n' + '# Defaults\n* text=auto eol=lf\n*.bat text eol=crlf\n' + '* text=auto eol=crlf\n* eol=lf\n' +) +no_lf_cases=( + '* text=auto\n*.ps1 text eol=crlf\n' + '* text=auto eol=crlf\n*.yml text eol=lf\n' + '* text=auto eol=lf\n* eol=crlf\n' + '# * text=auto eol=lf\n*.sh text eol=lf\n' +) +for attributes in "${lf_cases[@]}"; do + sync_bootstrap "${attributes}" "" + ! lf_warned || fail "LF as the last * eol must not warn: ${attributes}" +done +for attributes in "${no_lf_cases[@]}"; do + sync_bootstrap "${attributes}" "" + lf_warned || fail "a .gitattributes without an LF default must warn: ${attributes}" +done + +sync_bootstrap "" ".gitattributes" +[ ! -e "${consumer}/.gitattributes" ] || fail "an ignored .gitattributes must not be bootstrapped" +sync_bootstrap '* text=auto eol=crlf\n' ".gitattributes" +! lf_warned || fail "an ignored .gitattributes must not warn" + +echo "✓ config-sync: sync-files.sh bootstraps .gitattributes without overwriting it and warns when it has no LF default." From cef98a20b79392e2f66e321aa14edd38c112fd50 Mon Sep 17 00:00:00 2001 From: Jean-Paul van Ravensberg <14926452+DevSecNinja@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:26:31 +0100 Subject: [PATCH 2/4] feat(lint): fail on text files committed with CRLF line endings Refs #580 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/lint-include.yml | 12 +++++ .github/workflows/lint.yml | 58 +++++++++++++++++++++++ config-sync/templates/.gitattributes | 2 + config-sync/templates/README.md | 5 +- docs/architecture.md | 13 ++++++ scripts/lint/line-endings.sh | 69 ++++++++++++++++++++++++++++ tests/lint-include.py | 7 ++- tests/lint-job-modes-contract.sh | 2 +- tests/lint-scripts.sh | 40 ++++++++++++++++ workflow-templates/lint.yml | 4 ++ 10 files changed, 209 insertions(+), 3 deletions(-) create mode 100644 scripts/lint/line-endings.sh diff --git a/.github/workflows/lint-include.yml b/.github/workflows/lint-include.yml index 7a949b40..55122bff 100644 --- a/.github/workflows/lint-include.yml +++ b/.github/workflows/lint-include.yml @@ -151,6 +151,16 @@ on: required: false type: boolean default: false + lint-line-endings: + description: "Report text files committed with CRLF line endings (scripts/lint/line-endings.sh). See lint.yml." + required: false + type: boolean + default: false + lint-line-endings-blocking: + description: "Report CRLF line endings as errors and fail the line-endings linter; false reports warnings only" + required: false + type: boolean + default: true upload-sarif: description: "Upload checkov/trivy/zizmor/ruff/biome/psscriptanalyzer results to GitHub code scanning (requires GitHub Advanced Security). Set false for private repos without GHAS to gate on exit codes with console output instead." @@ -209,5 +219,7 @@ jobs: lint-config-drift: ${{ inputs.lint-config-drift }} lint-inline-scripts: ${{ inputs.lint-inline-scripts }} lint-inline-scripts-blocking: ${{ inputs.lint-inline-scripts-blocking }} + lint-line-endings: ${{ inputs.lint-line-endings }} + lint-line-endings-blocking: ${{ inputs.lint-line-endings-blocking }} upload-sarif: ${{ inputs.upload-sarif }} lint-job-mode: ${{ inputs.lint-job-mode }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5b48d736..13179dd0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -146,6 +146,19 @@ on: required: false type: boolean default: false + lint-line-endings: + description: >- + Report text files committed with CRLF or mixed line endings (scripts/lint/line-endings.sh, git ls-files --eol). + A path that needs CRLF gets its own .gitattributes rule: `eol=crlf` (stored with LF) or `-text` (stored byte + for byte). + required: false + type: boolean + default: true + lint-line-endings-blocking: + description: "Report CRLF line endings as errors and fail the line-endings linter; false reports warnings only" + required: false + type: boolean + default: true upload-sarif: description: "Upload checkov/trivy/zizmor/ruff/biome/psscriptanalyzer results to GitHub code scanning (requires GitHub Advanced Security). Set false for private repos without GHAS to gate on exit codes with console output instead." @@ -962,6 +975,43 @@ jobs: # The runner's system Python ships PyYAML; a caller's mise Python may not. run: /usr/bin/python3 "${RUNNER_TEMP}/devsecninja-lint/scripts/lint-inline-scripts.py" + line-endings: + name: line endings + if: ${{ inputs.lint-line-endings && (inputs.lint-job-mode == 'parallel' || (inputs.lint-job-mode == 'auto' && !github.event.repository.private)) + }} + continue-on-error: ${{ ! inputs.lint-fail-on-error }} + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Checkout lint scripts + # The linter scripts ship with this reusable workflow: fetch them from + # the same commit of DevSecNinja/.github that the caller pinned. + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} + path: .devsecninja-lint + sparse-checkout: |- + scripts/lint/ + scripts/lint-inline-scripts.py + sparse-checkout-cone-mode: false + persist-credentials: false + + - name: Move lint scripts out of the workspace + # Outside the workspace the caller's linters don't scan them, and the + # code checkout doesn't delete them. + run: mv .devsecninja-lint "${RUNNER_TEMP}/devsecninja-lint" + + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Check line endings + env: + LINT_LINE_ENDINGS_BLOCKING: ${{ inputs.lint-line-endings-blocking }} + run: bash "${RUNNER_TEMP}/devsecninja-lint/scripts/lint/line-endings.sh" + all-linters: # Single-job mode (`lint-job-mode: single`, or `auto` on a private # repository). Every enabled linter runs as a step of this one job, so a @@ -1289,6 +1339,14 @@ jobs: # The runner's system Python ships PyYAML; a caller's mise Python may not. run: /usr/bin/python3 "${RUNNER_TEMP}/devsecninja-lint/scripts/lint-inline-scripts.py" + - name: Check line endings + id: line-endings + if: ${{ inputs.lint-line-endings }} + continue-on-error: true + env: + LINT_LINE_ENDINGS_BLOCKING: ${{ inputs.lint-line-endings-blocking }} + run: bash "${RUNNER_TEMP}/devsecninja-lint/scripts/lint/line-endings.sh" + - name: Report linter results if: ${{ !cancelled() }} env: diff --git a/config-sync/templates/.gitattributes b/config-sync/templates/.gitattributes index 80505c21..f38dfaa7 100644 --- a/config-sync/templates/.gitattributes +++ b/config-sync/templates/.gitattributes @@ -9,6 +9,8 @@ # CRLF gets its own rule below, for example: # *.ps1 text eol=crlf CRLF in the checkout, LF in the repository # *.ics -text stored byte for byte (RFC 5545 calendars need CRLF) +# The shared Lint workflow (lint-line-endings) fails on text files committed +# with CRLF, except -text paths. * text=auto eol=lf # Scripts stay LF even when a repository changes the default above. diff --git a/config-sync/templates/README.md b/config-sync/templates/README.md index 997ebb4f..2c86c03e 100644 --- a/config-sync/templates/README.md +++ b/config-sync/templates/README.md @@ -74,7 +74,10 @@ uses another default lists `.gitattributes` in `.github/.config-sync-ignore`, which also skips that warning. Adding the file doesn't convert files already committed with CRLF; run -`git add --renormalize .` and commit the result. +`git add --renormalize .` and commit the result. The shared Lint workflow's +`lint-line-endings` check fails on text files committed with CRLF or mixed +line endings, except `-text` paths (see +[docs/architecture.md](../../docs/architecture.md#lint-lintyml)). ## renovate.json5 diff --git a/docs/architecture.md b/docs/architecture.md index 0807ff6a..6f723f05 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -254,6 +254,17 @@ runs it blocking. A later major release makes blocking the default. The checker runs with the runner's system Python, which ships PyYAML (`ubuntu-24.04`). +The `line endings` linter (`scripts/lint/line-endings.sh`) reports every text +file committed with CRLF or mixed line endings. It reads `git ls-files --eol`, +so git's own `.gitattributes` resolution decides: a path with `-text` (or +`binary`) is stored byte for byte and skipped, and an `eol=crlf` path passes, +because git stores it with LF and only checks it out with CRLF. Those rules are +the only way to keep CRLF; the linter has no exclude input. Findings are errors +that fail the linter; `lint-line-endings-blocking: false` reports them as +warnings. Fix them with `git add --renormalize .` once `.gitattributes` has +`* text=auto eol=lf`, which config-sync bootstraps (see +[Templates](#templates-config-synctemplates)). The linter needs only git. + | Input | Description | | ------------------------------ | ---------------------------------------------------------------------- | | `mise-version` | **Required.** mise version to install. | @@ -283,6 +294,8 @@ runner's system Python, which ships PyYAML (`ubuntu-24.04`). | `lint-config-drift` | Config-drift check. Default: `false`. | | `lint-inline-scripts` | Inline-script check (see above). Default: `true`. After v4.4.0. | | `lint-inline-scripts-blocking` | Report inline scripts as errors. Default: `false`. After v4.4.0. | +| `lint-line-endings` | CRLF line-endings check (see above). Default: `true`. After v4.7.0. | +| `lint-line-endings-blocking` | Report CRLF as errors and fail. Default: `true`. After v4.7.0. | **Example caller:** diff --git a/scripts/lint/line-endings.sh b/scripts/lint/line-endings.sh new file mode 100644 index 00000000..89cb3b33 --- /dev/null +++ b/scripts/lint/line-endings.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# Fails when a tracked text file is committed with CRLF (or mixed) line +# endings, using git's own attribute resolution (`git ls-files --eol`): +# a path whose .gitattributes rule unsets `text` (`-text`, or `binary`) is +# stored byte for byte and skipped, and an `eol=crlf` path is fine, because +# git stores it with LF and only checks it out with CRLF. +# A step of .github/workflows/lint.yml, run the same way in both lint modes. +# Env: LINT_LINE_ENDINGS_BLOCKING (true: errors, fail; false: warnings only). + +set -euo pipefail + +blocking="${LINT_LINE_ENDINGS_BLOCKING:-true}" +case "${blocking}" in +true) level=error ;; +false) level=warning ;; +*) + echo "::error::lint-line-endings-blocking must be true or false (got '${blocking}')." + exit 2 + ;; +esac + +# Escapes the message of a workflow command, and (escape_property) a +# property value such as file=.... +escape_data() { + local value="$1" + value="${value//%/%25}" + value="${value//$'\r'/%0D}" + value="${value//$'\n'/%0A}" + printf '%s' "${value}" +} +escape_property() { + local value + value="$(escape_data "$1")" + value="${value//:/%3A}" + value="${value//,/%2C}" + printf '%s' "${value}" +} + +max_annotations=50 +checked=0 +found=() +# Each record is "i/ w/ attr/\t". +while IFS=$'\t' read -r -d '' info path; do + checked=$((checked + 1)) + case "${info%% *}" in + i/crlf | i/mixed) ;; + *) continue ;; + esac + attributes="${info#*attr/}" + case " ${attributes} " in + *" -text "*) continue ;; + *) ;; + esac + found+=("${path}") +done < <(git ls-files --eol -z) + +if [ "${#found[@]}" -eq 0 ]; then + echo "✓ No text file is committed with CRLF line endings (${checked} files checked)." + exit 0 +fi + +for path in "${found[@]:0:${max_annotations}}"; do + echo "::${level} file=$(escape_property "${path}"),title=CRLF line endings::$(escape_data "${path}") is committed with CRLF line endings." +done +[ "${#found[@]}" -le "${max_annotations}" ] || + echo "… and $((${#found[@]} - max_annotations)) more file(s)." +echo "::${level} title=CRLF line endings::${#found[@]} text file(s) are committed with CRLF line endings. Store them with LF: with \"* text=auto eol=lf\" in .gitattributes, run \"git add --renormalize .\" and commit. A file that needs CRLF gets its own .gitattributes rule: \"eol=crlf\" (CRLF in the checkout, LF in the repository) or \"-text\" (stored byte for byte)." +[ "${blocking}" = false ] || exit 1 diff --git a/tests/lint-include.py b/tests/lint-include.py index 71de8551..35b2d4c9 100755 --- a/tests/lint-include.py +++ b/tests/lint-include.py @@ -35,7 +35,12 @@ LINT_REF = "$/.github/workflows/lint.yml" # Boolean `lint-*` inputs of lint.yml that are options rather than linters. -NOT_LINTERS = {"lint-fail-on-error", "lint-inline-scripts-blocking", "lint-biome-with-eslint"} +NOT_LINTERS = { + "lint-fail-on-error", + "lint-inline-scripts-blocking", + "lint-biome-with-eslint", + "lint-line-endings-blocking", +} LEVELS = {"none": 0, "read": 1, "write": 2} diff --git a/tests/lint-job-modes-contract.sh b/tests/lint-job-modes-contract.sh index ef32cb98..9c7ba233 100755 --- a/tests/lint-job-modes-contract.sh +++ b/tests/lint-job-modes-contract.sh @@ -14,7 +14,7 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" workflow="${repo_root}/.github/workflows/lint.yml" combined_job="all-linters" -linter_jobs=(dprint yamlfmt yamllint actionlint gitleaks go shellcheck shfmt checkov trivy zizmor ruff biome psscriptanalyzer config-drift inline-scripts) +linter_jobs=(dprint yamlfmt yamllint actionlint gitleaks go shellcheck shfmt checkov trivy zizmor ruff biome psscriptanalyzer config-drift inline-scripts line-endings) parallel_guard="(inputs.lint-job-mode == 'parallel' || (inputs.lint-job-mode == 'auto' && !github.event.repository.private))" fail() { diff --git a/tests/lint-scripts.sh b/tests/lint-scripts.sh index b3b4a89f..2a4811ab 100755 --- a/tests/lint-scripts.sh +++ b/tests/lint-scripts.sh @@ -473,4 +473,44 @@ expect_status 1 "drifted config" expect_out "::error::.yamllint.yaml has drifted from central config" "drifted config" expect_out "+central" "drifted config" +# line-endings fails on text files committed with CRLF or mixed line endings. +# -text and binary paths are stored byte for byte and skipped; eol=crlf paths +# pass, because git stores them with LF. +eol="${tmp_dir}/eol" +mkdir -p "${eol}/sub" +git -C "${eol}" init -q +printf 'a\nb\n' >"${eol}/lf.txt" +printf 'a\r\nb\r\n' >"${eol}/feed.ics" +printf 'a\r\n' >"${eol}/asset.dat" +printf 'a\0\r\n' >"${eol}/blob" +printf 'a\r\n' >"${eol}/run.bat" +printf '* text=auto eol=lf\n*.ics -text\n*.dat binary\n*.bat text eol=crlf\n' >"${eol}/.gitattributes" +git -C "${eol}" -c core.autocrlf=false add -A +run_in "${eol}" line-endings +expect_status 0 "LF, -text, binary and eol=crlf files" +expect_out "✓ No text file is committed with CRLF line endings (6 files checked)." "LF, -text, binary and eol=crlf files" + +# Files committed with CRLF before the .gitattributes rule, staged as is. +stage_raw() { + local sha + printf '%b' "$2" >"${eol}/$1" + sha="$(git -C "${eol}" hash-object -w --no-filters -- "$1")" + git -C "${eol}" update-index --add --cacheinfo "100644,${sha},$1" +} +stage_raw sub/crlf.txt 'a\r\nb\r\n' +stage_raw mixed.md 'a\nb\r\n' +stage_raw 'odd,name%.txt' 'a\r\n' +run_in "${eol}" line-endings +expect_status 1 "CRLF files" +expect_out "::error file=sub/crlf.txt,title=CRLF line endings::sub/crlf.txt is committed with CRLF line endings." "CRLF files" +expect_out "::error file=mixed.md,title=CRLF line endings::" "mixed line endings" +expect_out "::error file=odd%2Cname%25.txt,title=CRLF line endings::odd,name%25.txt is committed" "an escaped path" +expect_out "3 text file(s) are committed with CRLF line endings" "CRLF files" +! grep -Fq "feed.ics" <<<"${out}" || fail "a -text path must not be reported: ${out}" +run_in "${eol}" line-endings LINT_LINE_ENDINGS_BLOCKING=false +expect_status 0 "CRLF files without blocking" +expect_out "::warning file=sub/crlf.txt,title=CRLF line endings::" "CRLF files without blocking" +run_in "${eol}" line-endings LINT_LINE_ENDINGS_BLOCKING=bogus +expect_status 2 "an invalid blocking value" + echo "lint scripts: OK" diff --git a/workflow-templates/lint.yml b/workflow-templates/lint.yml index a03a2df2..3b7e90f8 100644 --- a/workflow-templates/lint.yml +++ b/workflow-templates/lint.yml @@ -82,3 +82,7 @@ jobs: # after v4.4.0; bump the pin above before uncommenting. # lint-inline-scripts: true # lint-inline-scripts-blocking: false # true: errors instead of warnings + # Optional: report text files committed with CRLF line endings. Needs a + # lint.yml release after v4.7.0; bump the pin above before uncommenting. + # lint-line-endings: true + # lint-line-endings-blocking: true # false: warnings instead of errors From a2b64e72fc76486843f3073ff86a40e2d0ed6bfa Mon Sep 17 00:00:00 2001 From: Jean-Paul van Ravensberg <14926452+DevSecNinja@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:39:45 +0100 Subject: [PATCH 3/4] fix(lint): keep tabs in paths and fail when the line-endings scan fails Split each git ls-files record at its first tab only, so paths keep their own tabs, and scan into a file so a failing git ls-files stops the linter instead of reporting no CRLF (Copilot review on #581). Carries the eol reset fix of the lower PR. Refs #580 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- config-sync/templates/README.md | 2 +- scripts/config-sync/sync-files.sh | 10 ++++++++-- scripts/lint/line-endings.sh | 14 +++++++++++--- tests/config-sync-scripts.sh | 3 +++ tests/lint-scripts.sh | 11 ++++++++++- 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/config-sync/templates/README.md b/config-sync/templates/README.md index 2c86c03e..596a6741 100644 --- a/config-sync/templates/README.md +++ b/config-sync/templates/README.md @@ -69,7 +69,7 @@ rule at the end, where later lines win: ``` Each sync run warns when an existing `.gitattributes` has no LF default (the -last `*` line that sets `eol` must set `eol=lf`). A repo that deliberately +last `*` line that sets or resets `eol` must set `eol=lf`). A repo that deliberately uses another default lists `.gitattributes` in `.github/.config-sync-ignore`, which also skips that warning. diff --git a/scripts/config-sync/sync-files.sh b/scripts/config-sync/sync-files.sh index 5117cac8..00aed195 100644 --- a/scripts/config-sync/sync-files.sh +++ b/scripts/config-sync/sync-files.sh @@ -197,12 +197,18 @@ else fi # .gitattributes is repo-owned as well; only check that LF stays the -# default line ending: the last `*` line that sets `eol` sets `eol=lf`. +# default line ending: the last `*` line that sets or resets `eol` sets +# `eol=lf` (`-eol`, `!eol` and a bare `eol` drop an earlier LF default). # A repository that needs another default lists .gitattributes in # its ignore file, which also skips this check. Exit 0: LF default. lf_default() { awk '{ sub(/\r$/, "") } - $1 == "*" { for (i = 2; i <= NF; i++) if ($i ~ /^eol=/) eol = substr($i, 5) } + $1 == "*" { + for (i = 2; i <= NF; i++) { + if ($i ~ /^eol=/) eol = substr($i, 5) + else if ($i == "eol" || $i == "-eol" || $i == "!eol") eol = "" + } + } END { exit eol != "lf" }' "$1" } # shellcheck disable=SC2310 # is_ignored and lf_default only test; their status is the answer. diff --git a/scripts/lint/line-endings.sh b/scripts/lint/line-endings.sh index 89cb3b33..fc38ba8b 100644 --- a/scripts/lint/line-endings.sh +++ b/scripts/lint/line-endings.sh @@ -40,9 +40,17 @@ escape_property() { max_annotations=50 checked=0 found=() -# Each record is "i/ w/ attr/\t". -while IFS=$'\t' read -r -d '' info path; do +# A file, not a process substitution: `set -e` then stops on a failing +# `git ls-files` (no repository, dubious ownership) instead of passing. +listing="$(mktemp)" +trap 'rm -f "${listing}"' EXIT +git ls-files --eol -z >"${listing}" +# Each record is "i/ w/ attr/\t"; +# split at the first tab only, so a path keeps its own tabs. +while IFS= read -r -d '' record; do checked=$((checked + 1)) + info="${record%%$'\t'*}" + path="${record#*$'\t'}" case "${info%% *}" in i/crlf | i/mixed) ;; *) continue ;; @@ -53,7 +61,7 @@ while IFS=$'\t' read -r -d '' info path; do *) ;; esac found+=("${path}") -done < <(git ls-files --eol -z) +done <"${listing}" if [ "${#found[@]}" -eq 0 ]; then echo "✓ No text file is committed with CRLF line endings (${checked} files checked)." diff --git a/tests/config-sync-scripts.sh b/tests/config-sync-scripts.sh index 34fcceb9..01a32f0c 100644 --- a/tests/config-sync-scripts.sh +++ b/tests/config-sync-scripts.sh @@ -133,11 +133,14 @@ lf_cases=( '* text=auto eol=lf\r\n*.ps1 text eol=crlf\r\n' '# Defaults\n* text=auto eol=lf\n*.bat text eol=crlf\n' '* text=auto eol=crlf\n* eol=lf\n' + '* text=auto eol=lf\n* !eol\n* eol=lf\n' ) no_lf_cases=( '* text=auto\n*.ps1 text eol=crlf\n' '* text=auto eol=crlf\n*.yml text eol=lf\n' '* text=auto eol=lf\n* eol=crlf\n' + '* text=auto eol=lf\n* !eol\n' + '* text=auto eol=lf\n* -eol\n' '# * text=auto eol=lf\n*.sh text eol=lf\n' ) for attributes in "${lf_cases[@]}"; do diff --git a/tests/lint-scripts.sh b/tests/lint-scripts.sh index 2a4811ab..da11936f 100755 --- a/tests/lint-scripts.sh +++ b/tests/lint-scripts.sh @@ -500,17 +500,26 @@ stage_raw() { stage_raw sub/crlf.txt 'a\r\nb\r\n' stage_raw mixed.md 'a\nb\r\n' stage_raw 'odd,name%.txt' 'a\r\n' +# A path with leading and trailing tabs (index only: not every file system allows them). +tab_sha="$(printf 'a\r\n' | git -C "${eol}" hash-object -w --no-filters --stdin)" +git -C "${eol}" -c core.protectNTFS=false update-index --add --cacheinfo "100644,${tab_sha},"$'\ttabs.txt\t' run_in "${eol}" line-endings expect_status 1 "CRLF files" expect_out "::error file=sub/crlf.txt,title=CRLF line endings::sub/crlf.txt is committed with CRLF line endings." "CRLF files" expect_out "::error file=mixed.md,title=CRLF line endings::" "mixed line endings" expect_out "::error file=odd%2Cname%25.txt,title=CRLF line endings::odd,name%25.txt is committed" "an escaped path" -expect_out "3 text file(s) are committed with CRLF line endings" "CRLF files" +expect_out "::error file="$'\ttabs.txt\t'",title=CRLF line endings::" "a path with tabs" +expect_out "4 text file(s) are committed with CRLF line endings" "CRLF files" ! grep -Fq "feed.ics" <<<"${out}" || fail "a -text path must not be reported: ${out}" run_in "${eol}" line-endings LINT_LINE_ENDINGS_BLOCKING=false expect_status 0 "CRLF files without blocking" expect_out "::warning file=sub/crlf.txt,title=CRLF line endings::" "CRLF files without blocking" run_in "${eol}" line-endings LINT_LINE_ENDINGS_BLOCKING=bogus expect_status 2 "an invalid blocking value" +# A failing scan fails the linter instead of reporting no CRLF. +mkdir -p "${tmp_dir}/not-a-repo" +run_in "${tmp_dir}/not-a-repo" line-endings GIT_CEILING_DIRECTORIES="${tmp_dir}" +expect_failure "outside a git repository" +! grep -Fq "No text file is committed" <<<"${out}" || fail "a failed scan must not report success: ${out}" echo "lint scripts: OK" From b21593bff7b3d2b112f31435367a54e075bb4749 Mon Sep 17 00:00:00 2001 From: Jean-Paul van Ravensberg <14926452+DevSecNinja@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:44:08 +0100 Subject: [PATCH 4/4] fix(config-sync): keep .gitattributes symlinks, accept a quoted catch-all A dangling .gitattributes symlink counts as existing, so the template isn't written through it; a quoted "*" catch-all counts as the LF default; the warning says to replace a conflicting catch-all (CodeRabbit review on #579). Refs #580 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- scripts/config-sync/sync-files.sh | 17 ++++++++++------- tests/config-sync-scripts.sh | 20 +++++++++++++++++--- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/scripts/config-sync/sync-files.sh b/scripts/config-sync/sync-files.sh index 00aed195..287e0856 100644 --- a/scripts/config-sync/sync-files.sh +++ b/scripts/config-sync/sync-files.sh @@ -151,7 +151,9 @@ sync_tree() { continue fi - if [ "${kind}" = "templates" ] && [ -e "${dest}" ]; then + # A symlink counts as existing even when it dangles: cp would write + # the template through it. + if [ "${kind}" = "templates" ] && { [ -e "${dest}" ] || [ -L "${dest}" ]; }; then echo " ✔ ${dest} already exists (templates are not overwritten)" continue fi @@ -197,13 +199,14 @@ else fi # .gitattributes is repo-owned as well; only check that LF stays the -# default line ending: the last `*` line that sets or resets `eol` sets -# `eol=lf` (`-eol`, `!eol` and a bare `eol` drop an earlier LF default). -# A repository that needs another default lists .gitattributes in -# its ignore file, which also skips this check. Exit 0: LF default. +# default line ending: the last catch-all (`*`, or the quoted `"*"`) that +# sets or resets `eol` sets `eol=lf` (`-eol`, `!eol` and a bare `eol` drop +# an earlier LF default). A repository that needs another default lists +# .gitattributes in its ignore file, which also skips this check. Exit 0: +# LF default. lf_default() { awk '{ sub(/\r$/, "") } - $1 == "*" { + $1 == "*" || $1 == "\"*\"" { for (i = 2; i <= NF; i++) { if ($i ~ /^eol=/) eol = substr($i, 5) else if ($i == "eol" || $i == "-eol" || $i == "!eol") eol = "" @@ -213,7 +216,7 @@ lf_default() { } # shellcheck disable=SC2310 # is_ignored and lf_default only test; their status is the answer. if [ -f .gitattributes ] && ! is_ignored .gitattributes && ! lf_default .gitattributes; then - echo "::warning file=.gitattributes,title=.gitattributes has no LF default::Add \"* text=auto eol=lf\" above the repo-specific rules, so every text file is stored and checked out with LF. Give a path that needs CRLF its own rule (\"eol=crlf\" or \"-text\"). See https://github.com/DevSecNinja/.github/blob/main/config-sync/templates/README.md#gitattributes" + echo "::warning file=.gitattributes,title=.gitattributes has no LF default::Make \"* text=auto eol=lf\" the last catch-all rule, before the path-specific rules: replace a catch-all that sets another eol (e.g. \"* text=auto eol=crlf\") instead of adding one above it. Every text file is then stored and checked out with LF. Give a path that needs CRLF its own rule (\"eol=crlf\" or \"-text\"). See https://github.com/DevSecNinja/.github/blob/main/config-sync/templates/README.md#gitattributes" fi # renovate.json5 is repo-owned; only check that its top-level diff --git a/tests/config-sync-scripts.sh b/tests/config-sync-scripts.sh index 01a32f0c..7c0f388d 100644 --- a/tests/config-sync-scripts.sh +++ b/tests/config-sync-scripts.sh @@ -93,8 +93,8 @@ echo "✓ config-sync: sync-files.sh keeps the ruff.toml and biome.json template gitattributes_template="${repo_root}/config-sync/templates/.gitattributes" # Runs the sync without sync-templates in a fresh consumer checkout. $1 is the -# content of its .gitattributes (empty: no file, printed with %b), $2 of its -# .config-sync-ignore. +# content of its .gitattributes (empty: no file, @symlink: a dangling symlink, +# otherwise printed with %b), $2 of its .config-sync-ignore. sync_bootstrap() { rm -rf "${consumer}" "${tmp_dir}/output" mkdir -p "${consumer}/.config-sync-source/config-sync" "${consumer}/.github" @@ -104,7 +104,12 @@ sync_bootstrap() { -c commit.gpgsign=false commit --quiet --allow-empty -m init # Already present, so .gitattributes is the only file that can change. printf '{ extends: ["github>DevSecNinja/.github//.renovate/default.json5"] }\n' >"${consumer}/renovate.json5" - [ -z "$1" ] || printf '%b' "$1" >"${consumer}/.gitattributes" + if [ "$1" = "@symlink" ]; then + # Where the file system allows symlinks: a dangling one. + ln -s missing/target "${consumer}/.gitattributes" 2>/dev/null || true + elif [ -n "$1" ]; then + printf '%b' "$1" >"${consumer}/.gitattributes" + fi [ -z "$2" ] || printf '%s\n' "$2" >"${consumer}/.github/.config-sync-ignore" ( cd "${consumer}" @@ -134,6 +139,7 @@ lf_cases=( '# Defaults\n* text=auto eol=lf\n*.bat text eol=crlf\n' '* text=auto eol=crlf\n* eol=lf\n' '* text=auto eol=lf\n* !eol\n* eol=lf\n' + '"*" text=auto eol=lf\n' ) no_lf_cases=( '* text=auto\n*.ps1 text eol=crlf\n' @@ -157,4 +163,12 @@ sync_bootstrap "" ".gitattributes" sync_bootstrap '* text=auto eol=crlf\n' ".gitattributes" ! lf_warned || fail "an ignored .gitattributes must not warn" +# A .gitattributes symlink is the repository's, even when it dangles. +sync_bootstrap "@symlink" "" +if [ -L "${consumer}/.gitattributes" ]; then + [ ! -e "${consumer}/missing" ] || + fail "a dangling .gitattributes symlink must not be written through: $(cat "${tmp_dir}/log")" + grep -Fxq updated=0 "${tmp_dir}/output" || fail "a .gitattributes symlink must not open a sync PR" +fi + echo "✓ config-sync: sync-files.sh bootstraps .gitattributes without overwriting it and warns when it has no LF default."