Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions .github/workflows/lint-include.yml
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,16 @@ on:
required: false
type: boolean
default: false
lint-line-endings:
description: "Report text files committed with CRLF line endings (scripts/lint/line-endings.sh). See lint.yml."
required: false
type: boolean
default: false
lint-line-endings-blocking:
description: "Report CRLF line endings as errors and fail the line-endings linter; false reports warnings only"
required: false
type: boolean
default: true
upload-sarif:
description: "Upload checkov/trivy/zizmor/ruff/biome/psscriptanalyzer results to GitHub code scanning (requires GitHub
Advanced Security). Set false for private repos without GHAS to gate on exit codes with console output instead."
Expand Down Expand Up @@ -209,5 +219,7 @@ jobs:
lint-config-drift: ${{ inputs.lint-config-drift }}
lint-inline-scripts: ${{ inputs.lint-inline-scripts }}
lint-inline-scripts-blocking: ${{ inputs.lint-inline-scripts-blocking }}
lint-line-endings: ${{ inputs.lint-line-endings }}
lint-line-endings-blocking: ${{ inputs.lint-line-endings-blocking }}
upload-sarif: ${{ inputs.upload-sarif }}
lint-job-mode: ${{ inputs.lint-job-mode }}
60 changes: 60 additions & 0 deletions .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,19 @@ on:
required: false
type: boolean
default: false
lint-line-endings:
description: >-
Report text files committed with CRLF or mixed line endings (scripts/lint/line-endings.sh, git ls-files --eol).
A path that needs CRLF gets its own .gitattributes rule: `eol=crlf` (stored with LF) or `-text` (stored byte
for byte).
required: false
type: boolean
default: true
lint-line-endings-blocking:
description: "Report CRLF line endings as errors and fail the line-endings linter; false reports warnings only"
required: false
type: boolean
default: true
upload-sarif:
description: "Upload checkov/trivy/zizmor/ruff/biome/psscriptanalyzer results to GitHub code scanning (requires GitHub
Advanced Security). Set false for private repos without GHAS to gate on exit codes with console output instead."
Expand Down Expand Up @@ -962,6 +975,43 @@ jobs:
# The runner's system Python ships PyYAML; a caller's mise Python may not.
run: /usr/bin/python3 "${RUNNER_TEMP}/devsecninja-lint/scripts/lint-inline-scripts.py"

line-endings:
name: line endings
if: ${{ inputs.lint-line-endings && (inputs.lint-job-mode == 'parallel' || (inputs.lint-job-mode == 'auto' && !github.event.repository.private))
}}
continue-on-error: ${{ ! inputs.lint-fail-on-error }}
runs-on: ubuntu-24.04
timeout-minutes: 10
steps:
- name: Checkout lint scripts
# The linter scripts ship with this reusable workflow: fetch them from
# the same commit of DevSecNinja/.github that the caller pinned.
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ job.workflow_repository }}
ref: ${{ job.workflow_sha }}
path: .devsecninja-lint
sparse-checkout: |-
scripts/lint/
scripts/lint-inline-scripts.py
sparse-checkout-cone-mode: false
persist-credentials: false

- name: Move lint scripts out of the workspace
# Outside the workspace the caller's linters don't scan them, and the
# code checkout doesn't delete them.
run: mv .devsecninja-lint "${RUNNER_TEMP}/devsecninja-lint"

- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false

- name: Check line endings
env:
LINT_LINE_ENDINGS_BLOCKING: ${{ inputs.lint-line-endings-blocking }}
run: bash "${RUNNER_TEMP}/devsecninja-lint/scripts/lint/line-endings.sh"

all-linters:
# Single-job mode (`lint-job-mode: single`, or `auto` on a private
# repository). Every enabled linter runs as a step of this one job, so a
Expand Down Expand Up @@ -1013,6 +1063,16 @@ jobs:
fetch-depth: ${{ !inputs.lint-gitleaks && 1 || 0 }}
persist-credentials: false

- name: Check line endings
# Needs only git: runs before the tool setup, so a failed mise
# install doesn't skip it.
id: line-endings
if: ${{ inputs.lint-line-endings }}
continue-on-error: true
env:
LINT_LINE_ENDINGS_BLOCKING: ${{ inputs.lint-line-endings-blocking }}
run: bash "${RUNNER_TEMP}/devsecninja-lint/scripts/lint/line-endings.sh"

- name: Find Python files
# Repositories without Python skip ruff (and its mise install).
id: python-files
Expand Down
2 changes: 2 additions & 0 deletions config-sync/templates/.gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
# CRLF gets its own rule below, for example:
# *.ps1 text eol=crlf CRLF in the checkout, LF in the repository
# *.ics -text stored byte for byte (RFC 5545 calendars need CRLF)
# The shared Lint workflow (lint-line-endings) fails on text files committed
# with CRLF, except -text paths.
Comment thread
DevSecNinja marked this conversation as resolved.
Resolved
* text=auto eol=lf

# Scripts stay LF even when a repository changes the default above.
Expand Down
5 changes: 4 additions & 1 deletion config-sync/templates/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,10 @@ uses another default lists `.gitattributes` in `.github/.config-sync-ignore`,
which also skips that warning.

Adding the file doesn't convert files already committed with CRLF; run
`git add --renormalize .` and commit the result.
`git add .gitattributes` and `git add --renormalize .`, and commit both. The
shared Lint workflow's `lint-line-endings` check fails on text files committed
with CRLF or mixed line endings, except `-text` paths (see
[docs/architecture.md](../../docs/architecture.md#lint-lintyml)).

## renovate.json5

Expand Down
15 changes: 15 additions & 0 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,19 @@ runs it blocking. A
later major release makes blocking the default. The checker runs with the
runner's system Python, which ships PyYAML (`ubuntu-24.04`).

The `line endings` linter (`scripts/lint/line-endings.sh`) reports every text
file committed with CRLF or mixed line endings. It reads `git ls-files --eol`,
so git's own `.gitattributes` resolution decides: a path with `-text` (or
`binary`) is stored byte for byte and skipped, and an `eol=crlf` path passes
once git stores it with LF (it only checks it out with CRLF); one still stored
with CRLF is reported like any other. Those rules are
the only way to keep CRLF; the linter has no exclude input. Findings are errors
that fail the linter (and the workflow only with `lint-fail-on-error: true`);
`lint-line-endings-blocking: false` reports them as warnings. To fix them, put
`* text=auto eol=lf` in `.gitattributes` (config-sync bootstraps it, see
[Templates](#templates-config-synctemplates)), run `git add .gitattributes` and
`git add --renormalize .`, and commit both. The linter needs only git.

**zizmor config.** After v4.7.0, zizmor runs with the shared
[`scripts/lint/zizmor.yml`](../scripts/lint/zizmor.yml), which ships with
`lint.yml` at the commit the caller pinned, unless the repository has its own
Expand Down Expand Up @@ -293,6 +306,8 @@ its own commit (`lint-include.yml` calling `lint.yml`), `./` in callers.
| `lint-config-drift` | Config-drift check. Default: `false`. |
| `lint-inline-scripts` | Inline-script check (see above). Default: `true`. After v4.4.0. |
| `lint-inline-scripts-blocking` | Report inline scripts as errors. Default: `false`. After v4.4.0. |
| `lint-line-endings` | CRLF line-endings check (see above). Default: `true`. After v4.7.0. |
| `lint-line-endings-blocking` | Report CRLF as errors and fail. Default: `true`. After v4.7.0. |

**Example caller:**

Expand Down
81 changes: 81 additions & 0 deletions scripts/lint/line-endings.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
#!/usr/bin/env bash
#
# Fails when a tracked text file is committed with CRLF (or mixed) line
# endings, using git's own attribute resolution (`git ls-files --eol`):
# a path whose .gitattributes rule unsets `text` (`-text`, or `binary`) is
# stored byte for byte and skipped. An `eol=crlf` path passes once git
# stores it with LF (it only checks it out with CRLF); one still stored
# with CRLF is reported, like any other, until `git add --renormalize`.
# A step of .github/workflows/lint.yml, run the same way in both lint modes.
# Env: LINT_LINE_ENDINGS_BLOCKING (true: errors, fail; false: warnings only).

set -euo pipefail
# Bytes, not characters: in a UTF-8 locale `read -d ''` can swallow the NUL
# after a path that ends in an incomplete multibyte sequence.
export LC_ALL=C

blocking="${LINT_LINE_ENDINGS_BLOCKING:-true}"
case "${blocking}" in
true) level=error ;;
false) level=warning ;;
*)
echo "::error::lint-line-endings-blocking must be true or false (got '${blocking}')."
exit 2
;;
esac

# Escapes the message of a workflow command, and (escape_property) a
# property value such as file=....
escape_data() {
local value="$1"
value="${value//%/%25}"
value="${value//$'\r'/%0D}"
value="${value//$'\n'/%0A}"
printf '%s' "${value}"
}
escape_property() {
local value
value="$(escape_data "$1")"
value="${value//:/%3A}"
value="${value//,/%2C}"
printf '%s' "${value}"
}

max_annotations=50
checked=0
found=()
# A file, not a process substitution: `set -e` then stops on a failing
# `git ls-files` (no repository, dubious ownership) instead of passing.
listing="$(mktemp)"
trap 'rm -f "${listing}"' EXIT
git ls-files --eol -z >"${listing}"
# Each record is "i/<index eol> w/<worktree eol> attr/<attributes>\t<path>";
# split at the first tab only, so a path keeps its own tabs.
while IFS= read -r -d '' record; do
Comment thread
DevSecNinja marked this conversation as resolved.
Resolved
checked=$((checked + 1))
info="${record%%$'\t'*}"
path="${record#*$'\t'}"
case "${info%% *}" in
i/crlf | i/mixed) ;;
*) continue ;;
esac
attributes="${info#*attr/}"
case " ${attributes} " in
*" -text "*) continue ;;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
*) ;;
esac
found+=("${path}")
done <"${listing}"

if [ "${#found[@]}" -eq 0 ]; then
echo "✓ No text file is committed with CRLF line endings (${checked} files checked)."
exit 0
fi

for path in "${found[@]:0:${max_annotations}}"; do
echo "::${level} file=$(escape_property "${path}"),title=CRLF line endings::$(escape_data "${path}") is committed with CRLF line endings."
done
[ "${#found[@]}" -le "${max_annotations}" ] ||
echo "… and $((${#found[@]} - max_annotations)) more file(s)."
echo "::${level} title=CRLF line endings::${#found[@]} text file(s) are committed with CRLF line endings. Store them with LF: put \"* text=auto eol=lf\" in .gitattributes, run \"git add .gitattributes\" and \"git add --renormalize .\", and commit both. A file that needs CRLF gets its own .gitattributes rule: \"eol=crlf\" (CRLF in the checkout, LF in the repository) or \"-text\" (stored byte for byte)."
[ "${blocking}" = false ] || exit 1
7 changes: 6 additions & 1 deletion tests/lint-include.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,12 @@

LINT_REF = "$/.github/workflows/lint.yml"
# Boolean `lint-*` inputs of lint.yml that are options rather than linters.
NOT_LINTERS = {"lint-fail-on-error", "lint-inline-scripts-blocking", "lint-biome-with-eslint"}
NOT_LINTERS = {
"lint-fail-on-error",
"lint-inline-scripts-blocking",
"lint-biome-with-eslint",
"lint-line-endings-blocking",
}
LEVELS = {"none": 0, "read": 1, "write": 2}


Expand Down
2 changes: 1 addition & 1 deletion tests/lint-job-modes-contract.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ set -euo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
workflow="${repo_root}/.github/workflows/lint.yml"
combined_job="all-linters"
linter_jobs=(dprint yamlfmt yamllint actionlint gitleaks go shellcheck shfmt checkov trivy zizmor ruff biome psscriptanalyzer config-drift inline-scripts)
linter_jobs=(dprint yamlfmt yamllint actionlint gitleaks go shellcheck shfmt checkov trivy zizmor ruff biome psscriptanalyzer config-drift inline-scripts line-endings)
parallel_guard="(inputs.lint-job-mode == 'parallel' || (inputs.lint-job-mode == 'auto' && !github.event.repository.private))"

fail() {
Expand Down
53 changes: 53 additions & 0 deletions tests/lint-scripts.sh
Original file line number Diff line number Diff line change
Expand Up @@ -482,4 +482,57 @@ expect_status 1 "drifted config"
expect_out "::error::.yamllint.yaml has drifted from central config" "drifted config"
expect_out "+central" "drifted config"

# line-endings fails on text files committed with CRLF or mixed line endings.
# -text and binary paths are stored byte for byte and skipped; eol=crlf paths
# pass, because git stores them with LF.
eol="${tmp_dir}/eol"
mkdir -p "${eol}/sub"
git -C "${eol}" init -q
printf 'a\nb\n' >"${eol}/lf.txt"
printf 'a\r\nb\r\n' >"${eol}/feed.ics"
printf 'a\r\n' >"${eol}/asset.dat"
printf 'a\0\r\n' >"${eol}/blob"
printf 'a\r\n' >"${eol}/run.bat"
printf '* text=auto eol=lf\n*.ics -text\n*.dat binary\n*.bat text eol=crlf\n' >"${eol}/.gitattributes"
git -C "${eol}" -c core.autocrlf=false add -A
run_in "${eol}" line-endings
expect_status 0 "LF, -text, binary and eol=crlf files"
expect_out "✓ No text file is committed with CRLF line endings (6 files checked)." "LF, -text, binary and eol=crlf files"

# Files committed with CRLF before the .gitattributes rule, staged as is.
stage_raw() {
local sha
printf '%b' "$2" >"${eol}/$1"
sha="$(git -C "${eol}" hash-object -w --no-filters -- "$1")"
git -C "${eol}" update-index --add --cacheinfo "100644,${sha},$1"
}
stage_raw sub/crlf.txt 'a\r\nb\r\n'
stage_raw mixed.md 'a\nb\r\n'
stage_raw 'odd,name%.txt' 'a\r\n'
# An eol=crlf path still stored with CRLF (committed before its rule) is reported too.
stage_raw legacy.bat 'a\r\n'
# A path with leading and trailing tabs (index only: not every file system allows them).
tab_sha="$(printf 'a\r\n' | git -C "${eol}" hash-object -w --no-filters --stdin)"
git -C "${eol}" -c core.protectNTFS=false update-index --add --cacheinfo "100644,${tab_sha},"$'\ttabs.txt\t'
run_in "${eol}" line-endings
expect_status 1 "CRLF files"
expect_out "::error file=sub/crlf.txt,title=CRLF line endings::sub/crlf.txt is committed with CRLF line endings." "CRLF files"
expect_out "::error file=mixed.md,title=CRLF line endings::" "mixed line endings"
expect_out "::error file=odd%2Cname%25.txt,title=CRLF line endings::odd,name%25.txt is committed" "an escaped path"
expect_out "::error file="$'\ttabs.txt\t'",title=CRLF line endings::" "a path with tabs"
expect_out "::error file=legacy.bat,title=CRLF line endings::" "an eol=crlf path stored with CRLF"
expect_out "5 text file(s) are committed with CRLF line endings" "CRLF files"
expect_out 'run "git add .gitattributes" and "git add --renormalize ."' "the fix"
! grep -Fq "feed.ics" <<<"${out}" || fail "a -text path must not be reported: ${out}"
run_in "${eol}" line-endings LINT_LINE_ENDINGS_BLOCKING=false
expect_status 0 "CRLF files without blocking"
expect_out "::warning file=sub/crlf.txt,title=CRLF line endings::" "CRLF files without blocking"
run_in "${eol}" line-endings LINT_LINE_ENDINGS_BLOCKING=bogus
expect_status 2 "an invalid blocking value"
# A failing scan fails the linter instead of reporting no CRLF.
mkdir -p "${tmp_dir}/not-a-repo"
run_in "${tmp_dir}/not-a-repo" line-endings GIT_CEILING_DIRECTORIES="${tmp_dir}"
expect_failure "outside a git repository"
! grep -Fq "No text file is committed" <<<"${out}" || fail "a failed scan must not report success: ${out}"

echo "lint scripts: OK"
4 changes: 4 additions & 0 deletions workflow-templates/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,3 +82,7 @@ jobs:
# after v4.4.0; bump the pin above before uncommenting.
# lint-inline-scripts: true
# lint-inline-scripts-blocking: false # true: errors instead of warnings
# Optional: report text files committed with CRLF line endings. Needs a
# lint.yml release after v4.7.0; bump the pin above before uncommenting.
# lint-line-endings: true
# lint-line-endings-blocking: true # false: warnings instead of errors
Loading