From 6dc12f53ea50b0e38730189071aa8d544396df23 Mon Sep 17 00:00:00 2001 From: yaswant <2984440+yaswant@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:42:00 +0100 Subject: [PATCH 1/6] Uniform copyrighter --- sbin/copyrighter | 395 ++++++++++++++++++++++++++++++++++++++ sbin/gh_add_user | 5 +- sbin/gh_manage_labels | 5 +- sbin/gh_manage_milestones | 5 +- 4 files changed, 401 insertions(+), 9 deletions(-) create mode 100755 sbin/copyrighter diff --git a/sbin/copyrighter b/sbin/copyrighter new file mode 100755 index 00000000..787f3803 --- /dev/null +++ b/sbin/copyrighter @@ -0,0 +1,395 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# (C) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +# ----------------------------------------------------------------------------- + +# copyrighter - insert or update the Met Office Crown copyright header in a +# file, or in every file under a directory. +# +# Usage: copyrighter [-t TYPE] [-r] [-n] [-h] [path...] + +set -euo pipefail + +readonly BORDER="-----------------------------------------------------------------------------" +readonly MARKER="Crown copyright Met Office" +readonly SUPPORTED_TYPES="python fortran cpp c shell rst markdown perl yaml json" + +# status symbols used to decorate every "copyrighter:" message +readonly SYM_ADDED="+" +readonly SYM_UPDATED="~" +readonly SYM_UNCHANGED="=" +readonly SYM_WARN="!" +readonly SYM_INFO="i" +readonly SYM_ERROR="✗" +readonly SYM_SUMMARY="»" + +FORCED_TYPE="" +RECURSIVE=0 +DRY_RUN=0 +ADDED_COUNT=0 +UPDATED_COUNT=0 +UNCHANGED_COUNT=0 +SKIPPED_COUNT=0 +HEADER_LINES=() +NEEDS_HEADER_FILES=() + +# prefer fd when available for directory traversal, falling back to find +# (ripgrep is not used for binary-file detection, see is_text_file) +HAVE_FD=0 +if command -v fd >/dev/null 2>&1; then HAVE_FD=1; fi + +usage() { + cat < [path...] + +Insert a Met Office Crown copyright header at the top of a file (after any +shebang line), or into every file under a directory. If a copyrighter header +is already present it is updated in place rather than duplicated. + +Options: + -t, --type TYPE Force the file type instead of auto-detecting it. + One of: ${SUPPORTED_TYPES} + -r, --recursive, --full + When a directory is given, descend into every nested + subdirectory (no depth limit). Without this, only the + files directly inside the given directory are processed. + -n, --dry-run, --check + Report what would change without modifying any file. + -h, --help Show this help message. + +File types are auto-detected from the file extension, falling back to the +shebang line for extension-less scripts. JSON has no native comment syntax, +so headers are inserted as "//" line comments (JSONC-style); a warning is +printed whenever this happens. + +Exits non-zero if any file needed a header added or updated, or if a path +could not be processed - useful for CI checks with --dry-run. +EOF +} + +# detect_type FILE - auto-detect a file's type from its extension, falling +# back to the shebang line when the extension is missing or unrecognised. +detect_type() { + local file="$1" ext first_line + + ext="${file##*.}" + [[ "$file" == *.* && "$ext" != "$file" ]] || ext="" + ext="${ext,,}" + + case "$ext" in + py) echo "python"; return ;; + f|f77|f90|f95|f03|f08|for) echo "fortran"; return ;; + c++|cc|cpp|cxx|h++|hh|hpp|hxx) echo "cpp"; return ;; + c|h) echo "c"; return ;; + bash|ksh|sh) echo "shell"; return ;; + rst) echo "rst"; return ;; + markdown|md) echo "markdown"; return ;; + pl|pm) echo "perl"; return ;; + yaml|yml) echo "yaml"; return ;; + json) echo "json"; return ;; + esac + + first_line=$(head -n 1 -- "$file" 2>/dev/null || true) + case "$first_line" in + '#!'*python*) echo "python"; return ;; + '#!'*perl*) echo "perl"; return ;; + '#!'*bash|'#!'*/sh|'#!'*ksh) echo "shell"; return ;; + esac + + echo "" +} + +# generate_header TYPE - populate the HEADER_LINES array with the +# comment-style-appropriate copyright block for TYPE. +generate_header() { + local type="$1" + local c_style=0 + + # Determine the comment character/style based on file type + case "$type" in + python|shell|yaml|perl) local p="#" ;; + fortran) local p="!" ;; + cpp|json) local p="//" ;; + c) local p=" *" ; c_style=1 ;; + rst) local p=" " ;; # Left-hand indent for body lines + markdown) local p="" ;; # No prefix inside HTML comments + *) echo "copyrighter: ${SYM_ERROR} internal error: unknown type '${type}'" >&2 + return 1 ;; + esac + + # Construct the core body of uniform copyright block + HEADER_LINES=( + "${p}${p:+ }${BORDER}" + "${p}${p:+ }(C) Crown copyright Met Office. All rights reserved." + "${p}${p:+ }The file LICENCE, distributed with this code, contains details of the terms" + "${p}${p:+ }under which the code may be used." + "${p}${p:+ }${BORDER}" + ) + + # Add language-specific wrapping for block-comment formats + if [ "$type" = "rst" ]; then + # Replace the first and last lines with the correct RST directive syntax + HEADER_LINES[0]=".. ${BORDER}" + HEADER_LINES[4]=" ${BORDER}" + elif [ "$type" = "markdown" ]; then + HEADER_LINES=("") + elif [ $c_style -eq 1 ]; then + HEADER_LINES=("/*" "${HEADER_LINES[@]}" " */") + fi +} + +# is_text_file FILE - true if FILE looks like text, false if binary +# +# always uses grep -I: unlike find/rg, ripgrep has no quiet flag that +# reliably reports binary-ness via its exit code (an empty pattern still +# "matches" binary files up to their first NUL byte), so using it here +# risks corrupting binaries by writing a text header into them +is_text_file() { + grep -Iq '' -- "$1" 2>/dev/null +} + +# collect_files DIR RECURSIVE - print a NUL-separated, sorted list of regular +# files under DIR, using fd when available and falling back to find +collect_files() { + local dir="$1" recursive="$2" + + if ((HAVE_FD)); then + local fd_opts=(--type f --hidden --no-ignore --absolute-path --print0) + ((recursive == 0)) && fd_opts+=(--max-depth 1) + fd "${fd_opts[@]}" . "$dir" | sort -z + else + local find_opts=(-type f) + ((recursive == 0)) && find_opts=(-maxdepth 1 "${find_opts[@]}") + find "$dir" "${find_opts[@]}" -print0 | sort -z + fi +} + +# Process_file FILE FORCED_TYPE - insert or update the copyright header in FILE. +process_file() { + local file="$1" forced_type="$2" type + + if ! is_text_file "$file"; then + echo "copyrighter: ${SYM_WARN} skipping '${file}' (binary file)" >&2 + ((++SKIPPED_COUNT)) + return 0 + fi + + type="${forced_type:-$(detect_type "$file")}" + if [[ -z "$type" ]]; then + echo "copyrighter: ${SYM_WARN} skipping '${file}' (unable to determine file type, use --type)" >&2 + ((++SKIPPED_COUNT)) + return 0 + fi + + if [[ "$type" == "json" ]]; then + echo "copyrighter: ${SYM_INFO} JSON has no comment syntax; '${file}' will become JSONC-style" >&2 + fi + + generate_header "$type" + local header_count=${#HEADER_LINES[@]} + + local lines=() + mapfile -t lines < "$file" + local nlines=${#lines[@]} + + local offset=0 + [[ "${lines[0]:-}" =~ ^#! ]] && offset=1 + + # find the relative position of the marker line within our own generated + # template (not file content), used to anchor the regex built below + local marker_rel=-1 i + for ((i = 0; i < header_count; i++)); do + if [[ "${HEADER_LINES[$i]}" == *"$MARKER"* ]]; then + marker_rel=$i + break + fi + done + + # match the marker line by prefix + a fixed regex body, tolerating an + # optional copyright year (e.g. "Met Office 2026."), so a pre-existing + # header isn't mistaken for new content and duplicated + local marker_prefix marker_escaped_prefix marker_regex + marker_prefix="${HEADER_LINES[$marker_rel]%%(C) Crown copyright*}" + marker_escaped_prefix=$(printf '%s' "$marker_prefix" | sed -e 's/[\\.^$*+?(){}|]/\\&/g') + marker_regex="^${marker_escaped_prefix}\(C\) Crown copyright Met Office( [0-9]{4})?\. All rights reserved\.$" + + local search_limit=$((offset + 100)) + ((search_limit > nlines)) && search_limit=$nlines + + # scan for every header-like block near the top of the file (not just the + # first) so that stale duplicates, e.g. left behind by an older version of + # this script, are all collapsed into a single canonical header + local block_starts=() block_ends=() + i=$offset + while ((i < search_limit)); do + if [[ "${lines[$i]}" =~ $marker_regex ]]; then + local candidate_start=$((i - marker_rel)) + local candidate_end=$((candidate_start + header_count - 1)) + if ((candidate_start >= offset && candidate_end < nlines)); then + block_starts+=("$candidate_start") + block_ends+=("$candidate_end") + i=$((candidate_end + 1)) + continue + fi + fi + ((++i)) + done + + local action="Added" + if ((${#block_starts[@]} == 1)); then + action="Updated" + local k matches=1 + for ((k = 0; k < header_count; k++)); do + if [[ "${lines[$((block_starts[0] + k))]}" != "${HEADER_LINES[$k]}" ]]; then + matches=0 + break + fi + done + ((matches)) && action="Unchanged" + elif ((${#block_starts[@]} > 1)); then + action="Updated" + fi + + if [[ "$action" == "Unchanged" ]]; then + echo "copyrighter: ${SYM_UNCHANGED} Unchanged ${type} header in '${file}' (already up to date)" + ((++UNCHANGED_COUNT)) + return 0 + fi + + # rebuild the file's content with every detected header block removed + local tail=() idx=$offset block_idx=0 + while ((idx < nlines)); do + if ((block_idx < ${#block_starts[@]} && idx == block_starts[block_idx])); then + idx=$((block_ends[block_idx] + 1)) + ((++block_idx)) + continue + fi + tail+=("${lines[$idx]}") + ((++idx)) + done + while ((${#tail[@]} > 0)) && [[ -z "${tail[0]}" ]]; do + tail=("${tail[@]:1}") + done + + local new_lines=() + ((offset > 0)) && new_lines+=("${lines[@]:0:offset}") + new_lines+=("${HEADER_LINES[@]}") + new_lines+=("") + ((${#tail[@]} > 0)) && new_lines+=("${tail[@]}") + + local symbol="$SYM_ADDED" + [[ "$action" == "Updated" ]] && symbol="$SYM_UPDATED" + NEEDS_HEADER_FILES+=("${file} (${action})") + [[ "$action" == "Added" ]] && ((++ADDED_COUNT)) || ((++UPDATED_COUNT)) + + if ((DRY_RUN)); then + echo "copyrighter: ${symbol} [dry-run] ${action} ${type} header in '${file}'" + return 0 + fi + + # Write the new header to a temporary file and replace the original file with it. + local perms tmpfile + perms=$(stat -c '%a' "$file" 2>/dev/null || echo "") + tmpfile="${file}.copyrighter.tmp.$$" + printf '%s\n' "${new_lines[@]}" > "$tmpfile" + [[ -n "$perms" ]] && chmod "$perms" "$tmpfile" + mv -- "$tmpfile" "$file" + + echo "copyrighter: ${symbol} ${action} ${type} header in '${file}'" +} + +# process_path PATH - dispatch a single CLI argument to a file or directory walk. +process_path() { + local target="$1" + + if [[ -f "$target" ]]; then + if [[ -n "$FORCED_TYPE" ]]; then + process_file "$target" "$FORCED_TYPE" + else + local detected + detected="$(detect_type "$target")" + if [[ -z "$detected" ]]; then + echo "copyrighter: ${SYM_WARN} unable to determine file type for '${target}', use --type" >&2 + ((++SKIPPED_COUNT)) + return 0 + fi + process_file "$target" "$detected" + fi + return 0 + fi + + if [[ -d "$target" ]]; then + local f + while IFS= read -r -d '' f; do + process_file "$f" "$FORCED_TYPE" + done < <(collect_files "$target" "$RECURSIVE") + return 0 + fi + + echo "copyrighter: ${SYM_ERROR} '${target}' does not exist" >&2 + return 1 +} + +main() { + local paths=() + + while (($# > 0)); do + case "$1" in + -t|--type) + [[ $# -ge 2 ]] || { echo "copyrighter: ${SYM_ERROR} --type requires an argument" >&2; exit 1; } + FORCED_TYPE="$2" + if [[ ! " ${SUPPORTED_TYPES} " == *" ${FORCED_TYPE} "* ]]; then + echo "copyrighter: ${SYM_ERROR} unknown type '${FORCED_TYPE}' (expected one of: ${SUPPORTED_TYPES})" >&2 + exit 1 + fi + shift 2 ;; + -r|--recursive|--full) + RECURSIVE=1 + shift ;; + -n|--dry-run|--check) + DRY_RUN=1 + shift ;; + -h|--help) + usage + exit 0 ;; + --) + shift + while (($# > 0)); do paths+=("$1"); shift; done ;; + -*) + echo "copyrighter: ${SYM_ERROR} unknown option '$1'" >&2 + usage >&2 + exit 1 ;; + *) + paths+=("$1") + shift ;; + esac + done + + if ((${#paths[@]} == 0)); then + usage >&2 + exit 1 + fi + + local rc=0 p + for p in "${paths[@]}"; do + process_path "$p" || rc=1 + done + + if ((${#NEEDS_HEADER_FILES[@]} > 0)); then + echo "copyrighter: ${SYM_SUMMARY} files needing a copyright header:" + local entry + for entry in "${NEEDS_HEADER_FILES[@]}"; do + echo " - ${entry}" + done + if ((DRY_RUN)); then rc=1; fi + fi + echo "" + echo " Run the script without -n|--dry-run|--check to actually apply changes." + echo "" + echo "copyrighter: ${SYM_SUMMARY} added ${ADDED_COUNT}, updated ${UPDATED_COUNT}, unchanged ${UNCHANGED_COUNT}, skipped ${SKIPPED_COUNT}" + exit "$rc" +} + +main "$@" diff --git a/sbin/gh_add_user b/sbin/gh_add_user index d24d66a1..469696f2 100755 --- a/sbin/gh_add_user +++ b/sbin/gh_add_user @@ -1,10 +1,9 @@ #!/usr/bin/env bash - -# ---------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # (C) Crown copyright Met Office. All rights reserved. # The file LICENCE, distributed with this code, contains details of the terms # under which the code may be used. -# ---------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Add collaborator to a repository # gh api --method PUT /repos/:owner/:repo/collaborators/:username diff --git a/sbin/gh_manage_labels b/sbin/gh_manage_labels index c5229944..71355ff5 100755 --- a/sbin/gh_manage_labels +++ b/sbin/gh_manage_labels @@ -1,10 +1,9 @@ #!/usr/bin/env bash - -# ---------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # (C) Crown copyright Met Office. All rights reserved. # The file LICENCE, distributed with this code, contains details of the terms # under which the code may be used. -# ---------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Script to create standard labels in multiple GitHub repositories in parallel # Requires: diff --git a/sbin/gh_manage_milestones b/sbin/gh_manage_milestones index f2fd3b6b..ce8f35e9 100755 --- a/sbin/gh_manage_milestones +++ b/sbin/gh_manage_milestones @@ -1,10 +1,9 @@ #!/usr/bin/env bash - -# ---------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # (C) Crown copyright Met Office. All rights reserved. # The file LICENCE, distributed with this code, contains details of the terms # under which the code may be used. -# ---------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Script to create, update and close milestones in multiple GitHub repositories # Requires GitHub CLI: https://cli.github.com/ and Admin privileges to the repos From af31d26e41c70fc23259987aedcf6c5ce54acba9 Mon Sep 17 00:00:00 2001 From: yaswant <2984440+yaswant@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:43:34 +0100 Subject: [PATCH 2/6] sort file extensions --- sbin/copyrighter | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sbin/copyrighter b/sbin/copyrighter index 787f3803..f91ea602 100755 --- a/sbin/copyrighter +++ b/sbin/copyrighter @@ -79,16 +79,16 @@ detect_type() { ext="${ext,,}" case "$ext" in - py) echo "python"; return ;; - f|f77|f90|f95|f03|f08|for) echo "fortran"; return ;; + bash|ksh|sh) echo "shell"; return ;; c++|cc|cpp|cxx|h++|hh|hpp|hxx) echo "cpp"; return ;; c|h) echo "c"; return ;; - bash|ksh|sh) echo "shell"; return ;; - rst) echo "rst"; return ;; + f|f77|f90|f95|f03|f08|for) echo "fortran"; return ;; + json) echo "json"; return ;; markdown|md) echo "markdown"; return ;; pl|pm) echo "perl"; return ;; + py) echo "python"; return ;; + rst) echo "rst"; return ;; yaml|yml) echo "yaml"; return ;; - json) echo "json"; return ;; esac first_line=$(head -n 1 -- "$file" 2>/dev/null || true) From 61babf7b30a8b346ca01b321682c5bdfb78779a3 Mon Sep 17 00:00:00 2001 From: yaswant <2984440+yaswant@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:54:06 +0100 Subject: [PATCH 3/6] Fix shellcheck warning --- sbin/copyrighter | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sbin/copyrighter b/sbin/copyrighter index f91ea602..7443824c 100755 --- a/sbin/copyrighter +++ b/sbin/copyrighter @@ -282,7 +282,11 @@ process_file() { local symbol="$SYM_ADDED" [[ "$action" == "Updated" ]] && symbol="$SYM_UPDATED" NEEDS_HEADER_FILES+=("${file} (${action})") - [[ "$action" == "Added" ]] && ((++ADDED_COUNT)) || ((++UPDATED_COUNT)) + if [[ "$action" == "Added" ]]; then + ((++ADDED_COUNT)) + else + ((++UPDATED_COUNT)) + fi if ((DRY_RUN)); then echo "copyrighter: ${symbol} [dry-run] ${action} ${type} header in '${file}'" From 86b19bd98088a42dcd34f37e66fe2c3d9709a112 Mon Sep 17 00:00:00 2001 From: yaswant <2984440+yaswant@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:37:48 +0100 Subject: [PATCH 4/6] Add ignore option and expand supported file types --- sbin/copyrighter | 74 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 9 deletions(-) diff --git a/sbin/copyrighter b/sbin/copyrighter index 7443824c..343d195a 100755 --- a/sbin/copyrighter +++ b/sbin/copyrighter @@ -8,13 +8,14 @@ # copyrighter - insert or update the Met Office Crown copyright header in a # file, or in every file under a directory. # -# Usage: copyrighter [-t TYPE] [-r] [-n] [-h] [path...] +# Usage: copyrighter [-t TYPE] [-i TYPE] [-r] [-n] [-h] [path...] set -euo pipefail readonly BORDER="-----------------------------------------------------------------------------" readonly MARKER="Crown copyright Met Office" -readonly SUPPORTED_TYPES="python fortran cpp c shell rst markdown perl yaml json" +readonly SUPPORTED_TYPES="python fortran cpp c shell rst markdown perl yaml json toml rust ruby go js latex awk java powershell r postscript" +readonly VCS_DIRS=(.git .svn) # status symbols used to decorate every "copyrighter:" message readonly SYM_ADDED="+" @@ -26,6 +27,7 @@ readonly SYM_ERROR="✗" readonly SYM_SUMMARY="»" FORCED_TYPE="" +IGNORED_TYPES=() RECURSIVE=0 DRY_RUN=0 ADDED_COUNT=0 @@ -51,6 +53,9 @@ is already present it is updated in place rather than duplicated. Options: -t, --type TYPE Force the file type instead of auto-detecting it. One of: ${SUPPORTED_TYPES} + -i, --ignore TYPE Skip files whose (auto-detected or forced) type matches. + Accepts a comma-separated list and may be given more + than once; repeats and lists accumulate. -r, --recursive, --full When a directory is given, descend into every nested subdirectory (no depth limit). Without this, only the @@ -79,15 +84,26 @@ detect_type() { ext="${ext,,}" case "$ext" in + awk) echo "awk"; return ;; bash|ksh|sh) echo "shell"; return ;; c++|cc|cpp|cxx|h++|hh|hpp|hxx) echo "cpp"; return ;; c|h) echo "c"; return ;; f|f77|f90|f95|f03|f08|for) echo "fortran"; return ;; + go) echo "go"; return ;; + java) echo "java"; return ;; + js|cjs|mjs) echo "js"; return ;; json) echo "json"; return ;; markdown|md) echo "markdown"; return ;; pl|pm) echo "perl"; return ;; + ps1|psm1|psd1) echo "powershell"; return ;; + ps|eps) echo "postscript"; return ;; py) echo "python"; return ;; + r) echo "r"; return ;; + rb) echo "ruby"; return ;; + rs) echo "rust"; return ;; rst) echo "rst"; return ;; + tex) echo "latex"; return ;; + toml) echo "toml"; return ;; yaml|yml) echo "yaml"; return ;; esac @@ -95,6 +111,7 @@ detect_type() { case "$first_line" in '#!'*python*) echo "python"; return ;; '#!'*perl*) echo "perl"; return ;; + '#!'*ruby*) echo "ruby"; return ;; '#!'*bash|'#!'*/sh|'#!'*ksh) echo "shell"; return ;; esac @@ -109,9 +126,10 @@ generate_header() { # Determine the comment character/style based on file type case "$type" in - python|shell|yaml|perl) local p="#" ;; + python|shell|yaml|perl|toml|ruby|awk|powershell|r) local p="#" ;; fortran) local p="!" ;; - cpp|json) local p="//" ;; + cpp|json|java|rust|go|js) local p="//" ;; + latex|postscript) local p="%" ;; c) local p=" *" ; c_style=1 ;; rst) local p=" " ;; # Left-hand indent for body lines markdown) local p="" ;; # No prefix inside HTML comments @@ -150,19 +168,35 @@ is_text_file() { grep -Iq '' -- "$1" 2>/dev/null } +# is_type_ignored TYPE - true if TYPE was named via -i/--ignore +is_type_ignored() { + local type="$1" ignored + for ignored in "${IGNORED_TYPES[@]}"; do + [[ "$ignored" == "$type" ]] && return 0 + done + return 1 +} + # collect_files DIR RECURSIVE - print a NUL-separated, sorted list of regular -# files under DIR, using fd when available and falling back to find +# files under DIR, using fd when available and falling back to find; version +# control directories (VCS_DIRS) are always excluded from the walk collect_files() { local dir="$1" recursive="$2" if ((HAVE_FD)); then local fd_opts=(--type f --hidden --no-ignore --absolute-path --print0) + local vcs_dir + for vcs_dir in "${VCS_DIRS[@]}"; do fd_opts+=(--exclude "$vcs_dir"); done ((recursive == 0)) && fd_opts+=(--max-depth 1) fd "${fd_opts[@]}" . "$dir" | sort -z else - local find_opts=(-type f) - ((recursive == 0)) && find_opts=(-maxdepth 1 "${find_opts[@]}") - find "$dir" "${find_opts[@]}" -print0 | sort -z + local prune_expr=() vcs_dir maxdepth_opts=() + for vcs_dir in "${VCS_DIRS[@]}"; do + ((${#prune_expr[@]} > 0)) && prune_expr+=(-o) + prune_expr+=(-name "$vcs_dir") + done + ((recursive == 0)) && maxdepth_opts=(-maxdepth 1) + find "$dir" "${maxdepth_opts[@]}" \( "${prune_expr[@]}" \) -prune -o -type f -print0 | sort -z fi } @@ -183,6 +217,12 @@ process_file() { return 0 fi + if is_type_ignored "$type"; then + echo "copyrighter: ${SYM_INFO} skipping '${file}' (ignored type '${type}')" >&2 + ((++SKIPPED_COUNT)) + return 0 + fi + if [[ "$type" == "json" ]]; then echo "copyrighter: ${SYM_INFO} JSON has no comment syntax; '${file}' will become JSONC-style" >&2 fi @@ -195,7 +235,11 @@ process_file() { local nlines=${#lines[@]} local offset=0 - [[ "${lines[0]:-}" =~ ^#! ]] && offset=1 + if [[ "${lines[0]:-}" =~ ^#! ]]; then + offset=1 + elif [[ "$type" == "postscript" && "${lines[0]:-}" =~ ^%! ]]; then + offset=1 # preserve the required %!PS-Adobe magic line + fi # find the relative position of the marker line within our own generated # template (not file content), used to anchor the regex built below @@ -349,6 +393,18 @@ main() { exit 1 fi shift 2 ;; + -i|--ignore) + [[ $# -ge 2 ]] || { echo "copyrighter: ${SYM_ERROR} --ignore requires an argument" >&2; exit 1; } + local ignore_arg ignore_list + IFS=',' read -r -a ignore_list <<< "$2" + for ignore_arg in "${ignore_list[@]}"; do + if [[ ! " ${SUPPORTED_TYPES} " == *" ${ignore_arg} "* ]]; then + echo "copyrighter: ${SYM_ERROR} unknown type '${ignore_arg}' (expected one of: ${SUPPORTED_TYPES})" >&2 + exit 1 + fi + IGNORED_TYPES+=("$ignore_arg") + done + shift 2 ;; -r|--recursive|--full) RECURSIVE=1 shift ;; From 27d21a2991bb4fbd3128e70776d8c0bbac4b8ad6 Mon Sep 17 00:00:00 2001 From: yaswant <2984440+yaswant@users.noreply.github.com> Date: Sat, 10 Oct 2026 02:23:54 +0100 Subject: [PATCH 5/6] Rewrite copyrighter in Python for better performance on lustre --- bin/__init__.py | 0 bin/copyrighter.py | 411 ++++++++++++++++++++++++++ bin/tests/__init__.py | 0 bin/tests/test_copyrighter.py | 524 ++++++++++++++++++++++++++++++++++ sbin/copyrighter | 455 ----------------------------- 5 files changed, 935 insertions(+), 455 deletions(-) create mode 100644 bin/__init__.py create mode 100755 bin/copyrighter.py create mode 100644 bin/tests/__init__.py create mode 100644 bin/tests/test_copyrighter.py delete mode 100755 sbin/copyrighter diff --git a/bin/__init__.py b/bin/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/bin/copyrighter.py b/bin/copyrighter.py new file mode 100755 index 00000000..a967ca94 --- /dev/null +++ b/bin/copyrighter.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 +# ----------------------------------------------------------------------------- +# (C) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +# ----------------------------------------------------------------------------- +"""copyrighter - insert or update the Met Office Crown copyright header in a +file, or in every file under a directory. + +Standalone Python replica of sbin/copyrighter (bash). Can be run directly +(it is executable and carries its own shebang) or via `python3 copyrighter.py`. +""" + +from __future__ import annotations + +import argparse +import os +import re +import stat +import sys +import tempfile +from pathlib import Path + +BORDER = "-" * 77 +MARKER = "Crown copyright Met Office" + +# version-control directories that are always pruned from directory walks +VCS_DIRS = frozenset({".git", ".svn"}) + +EXTENSION_TO_TYPE: dict[str, str] = { + "awk": "awk", + "bash": "shell", "ksh": "shell", "sh": "shell", + "c++": "cpp", "cc": "cpp", "cpp": "cpp", "cxx": "cpp", + "h++": "cpp", "hh": "cpp", "hpp": "cpp", "hxx": "cpp", + "c": "c", "h": "c", + "f90": "fortran", "f95": "fortran", "f03": "fortran", "f08": "fortran", + "f": "fortran77", "f77": "fortran77", "for": "fortran77", + "go": "go", + "java": "java", + "js": "js", "cjs": "js", "mjs": "js", + "json": "json", + "markdown": "markdown", "md": "markdown", + "pl": "perl", "pm": "perl", + "ps1": "powershell", "psm1": "powershell", "psd1": "powershell", + "ps": "postscript", "eps": "postscript", + "py": "python", + "r": "r", + "rb": "ruby", + "rs": "rust", + "rst": "rst", + "tex": "latex", + "toml": "toml", + "yaml": "yaml", "yml": "yaml", +} + +# comment-prefix per type; "c", "rst" and "markdown" get extra wrapping below +COMMENT_PREFIX: dict[str, str] = { + "python": "#", "shell": "#", "yaml": "#", "perl": "#", "toml": "#", + "ruby": "#", "awk": "#", "powershell": "#", "r": "#", + "fortran": "!", + "fortran77": "C", # fixed-form F77: marker must be in column 1 + "cpp": "//", "json": "//", "java": "//", "rust": "//", "go": "//", "js": "//", + "latex": "%", "postscript": "%", + "c": " *", + "rst": " ", + "markdown": "", +} + +SUPPORTED_TYPES = tuple(sorted(set(COMMENT_PREFIX))) + +SYM = { + "added": "+", "updated": "~", "unchanged": "=", + "warn": "!", "info": "i", "error": "\u2717", "summary": "\u00bb", +} + + +# per-type separator between the comment prefix and the header text; only +# fortran77 (column 1 marker + 5 spaces, matching the classic column-7 +# statement field) and markdown (no prefix at all) differ from a single space +COMMENT_SEP: dict[str, str] = {"fortran77": " ", "markdown": ""} + + +def generate_header(file_type: str) -> list[str]: + """Return the comment-style-appropriate copyright block for file_type.""" + prefix = COMMENT_PREFIX[file_type] + sep = COMMENT_SEP.get(file_type, " ") + lines = [ + f"{prefix}{sep}{BORDER}", + f"{prefix}{sep}(C) Crown copyright Met Office. All rights reserved.", + ( + f"{prefix}{sep}The file LICENCE, distributed with this code, contains " + "details of the terms" + ), + f"{prefix}{sep}under which the code may be used.", + f"{prefix}{sep}{BORDER}", + ] + if file_type == "rst": + lines[0] = f".. {BORDER}" + lines[4] = f" {BORDER}" + elif file_type == "markdown": + lines = [""] + elif file_type == "c": + lines = ["/*", *lines, " */"] + return lines + + +def _extension(name: str) -> str: + base = os.path.basename(name) + if "." not in base: + return "" + ext = base.rsplit(".", 1)[1] + return "" if ext == base else ext.lower() + + +def _shebang_type(first_line: str) -> str: + if not first_line.startswith("#!"): + return "" + if "python" in first_line: + return "python" + if "perl" in first_line: + return "perl" + if "ruby" in first_line: + return "ruby" + if first_line.endswith("bash") or first_line.endswith("/sh") or first_line.endswith("ksh"): + return "shell" + return "" + + +def detect_type(name: str, first_line: str) -> str: + """Detect a file's type from its extension, falling back to the shebang.""" + file_type = EXTENSION_TO_TYPE.get(_extension(name)) + return file_type if file_type else _shebang_type(first_line) + + +def split_lines(text: str) -> list[str]: + """Split on '\\n' only (matching bash `mapfile`), dropping one trailing + empty element caused by a final newline, never the file content itself.""" + if text == "": + return [] + text = text.removesuffix("\n") + return text.split("\n") + + +def collect_files(directory: Path, recursive: bool) -> list[Path]: + """List regular files under directory, excluding VCS_DIRS and symlinks + (matching the `find`/`fd -type f` semantics used by the bash original).""" + results: list[Path] = [] + if recursive: + for root, dirnames, filenames in os.walk(directory): + dirnames[:] = [d for d in dirnames if d not in VCS_DIRS] + for fname in filenames: + full = Path(root) / fname + if not full.is_symlink() and full.is_file(): + results.append(full) + else: + for entry in directory.iterdir(): + if not entry.is_symlink() and entry.is_file(): + results.append(entry) + results.sort(key=str) + return results + + +class Counts: + def __init__(self) -> None: + self.added = 0 + self.updated = 0 + self.unchanged = 0 + self.skipped = 0 + + +def atomic_write(path: Path, data: bytes) -> None: + """Write data to path atomically, preserving the original file's exact + permission bits, using a securely-created (O_EXCL) temp file to avoid a + symlink race on the temporary path.""" + mode = stat.S_IMODE(path.stat().st_mode) + fd, tmp_name = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.copyrighter.") + try: + with os.fdopen(fd, "wb") as fh: + fh.write(data) + os.chmod(tmp_name, mode) + os.replace(tmp_name, path) + except BaseException: + try: + os.unlink(tmp_name) + except OSError: + pass + raise + + +def process_file( + path: Path, + forced_type: str | None, + ignored_types: set[str], + counts: Counts, + dry_run: bool, + needs_header: list[str], +) -> None: + try: + raw = path.read_bytes() + except OSError as exc: + print(f"copyrighter: {SYM['warn']} skipping '{path}' (cannot read: {exc.strerror})", file=sys.stderr) + counts.skipped += 1 + return + + if b"\x00" in raw: + print(f"copyrighter: {SYM['warn']} skipping '{path}' (binary file)", file=sys.stderr) + counts.skipped += 1 + return + + text = raw.decode("utf-8", errors="surrogateescape") + lines = split_lines(text) + first_line = lines[0] if lines else "" + + file_type = forced_type or detect_type(str(path), first_line) + if not file_type: + print(f"copyrighter: {SYM['warn']} skipping '{path}' (unable to determine file type, use --type)", file=sys.stderr) + counts.skipped += 1 + return + + if file_type in ignored_types: + print(f"copyrighter: {SYM['info']} skipping '{path}' (ignored type '{file_type}')", file=sys.stderr) + counts.skipped += 1 + return + + if file_type == "json": + print(f"copyrighter: {SYM['info']} JSON has no comment syntax; '{path}' will become JSONC-style", file=sys.stderr) + + header_lines = generate_header(file_type) + header_count = len(header_lines) + nlines = len(lines) + + offset = 0 + if first_line.startswith("#!"): + offset = 1 + elif file_type == "postscript" and first_line.startswith("%!"): + offset = 1 # preserve the required %!PS-Adobe magic line + + marker_rel = next((i for i, hl in enumerate(header_lines) if MARKER in hl), -1) + marker_prefix = header_lines[marker_rel].split("(C) Crown copyright", 1)[0] + marker_re = re.compile( + re.escape(marker_prefix) + r"\(C\) Crown copyright Met Office(?: \d{4})?\. All rights reserved\." + ) + + search_limit = min(offset + 100, nlines) + + # scan for every header-like block near the top of the file (not just the + # first) so stale duplicates are all collapsed into a single canonical header + block_starts: list[int] = [] + block_ends: list[int] = [] + i = offset + while i < search_limit: + if marker_re.fullmatch(lines[i]): + candidate_start = i - marker_rel + candidate_end = candidate_start + header_count - 1 + if candidate_start >= offset and candidate_end < nlines: + block_starts.append(candidate_start) + block_ends.append(candidate_end) + i = candidate_end + 1 + continue + i += 1 + + action = "Added" + if len(block_starts) == 1: + start = block_starts[0] + action = "Unchanged" if lines[start:start + header_count] == header_lines else "Updated" + elif len(block_starts) > 1: + action = "Updated" + + if action == "Unchanged": + print(f"copyrighter: {SYM['unchanged']} Unchanged {file_type} header in '{path}' (already up to date)") + counts.unchanged += 1 + return + + # rebuild the file's content with every detected header block removed + tail: list[str] = [] + idx = offset + block_idx = 0 + while idx < nlines: + if block_idx < len(block_starts) and idx == block_starts[block_idx]: + idx = block_ends[block_idx] + 1 + block_idx += 1 + continue + tail.append(lines[idx]) + idx += 1 + while tail and tail[0] == "": + tail.pop(0) + + new_lines = lines[:offset] + header_lines + [""] + tail + + symbol = SYM["updated"] if action == "Updated" else SYM["added"] + needs_header.append(f"{path} ({action})") + if action == "Added": + counts.added += 1 + else: + counts.updated += 1 + + if dry_run: + print(f"copyrighter: {symbol} [dry-run] {action} {file_type} header in '{path}'") + return + + new_text = "\n".join(new_lines) + "\n" + atomic_write(path, new_text.encode("utf-8", errors="surrogateescape")) + + print(f"copyrighter: {symbol} {action} {file_type} header in '{path}'") + + +def process_path( + target: str, + forced_type: str | None, + ignored_types: set[str], + recursive: bool, + counts: Counts, + dry_run: bool, + needs_header: list[str], +) -> int: + path = Path(target) + + if path.is_file(): + process_file(path, forced_type, ignored_types, counts, dry_run, needs_header) + return 0 + + if path.is_dir(): + for f in collect_files(path, recursive): + process_file(f, forced_type, ignored_types, counts, dry_run, needs_header) + return 0 + + print(f"copyrighter: {SYM['error']} '{target}' does not exist", file=sys.stderr) + return 1 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="copyrighter.py", + formatter_class=argparse.RawDescriptionHelpFormatter, + description=( + "Insert a Met Office Crown copyright header at the top of a file " + "(after any shebang line), or into every file under a directory. " + "If a copyrighter header is already present it is updated in " + "place rather than duplicated." + ), + epilog=( + "File types are auto-detected from the file extension, falling back\n" + "to the shebang line for extension-less scripts. JSON has no native\n" + 'comment syntax, so headers are inserted as "//" line comments\n' + "(JSONC-style); a warning is printed whenever this happens.\n\n" + "Exits non-zero if any file needed a header added or updated, or if\n" + "a path could not be processed - useful for CI checks with --dry-run." + ), + ) + parser.add_argument("paths", nargs="+", metavar="file-or-directory") + parser.add_argument( + "-t", "--type", dest="forced_type", choices=SUPPORTED_TYPES, metavar="TYPE", + help="Force the file type instead of auto-detecting it.", + ) + parser.add_argument( + "-i", "--ignore", dest="ignore", action="append", default=[], metavar="TYPE", + help="Skip files whose (auto-detected or forced) type matches. Accepts " + "a comma-separated list and may be given more than once.", + ) + parser.add_argument( + "-r", "--recursive", "--full", dest="recursive", action="store_true", + help="When a directory is given, descend into every nested subdirectory.", + ) + parser.add_argument( + "-n", "--dry-run", "--check", dest="dry_run", action="store_true", + help="Report what would change without modifying any file.", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + + ignored_types: set[str] = set() + for group in args.ignore: + for item in group.split(","): + item = item.strip() + if not item: + continue + if item not in SUPPORTED_TYPES: + parser.error(f"unknown type '{item}' (expected one of: {' '.join(SUPPORTED_TYPES)})") + ignored_types.add(item) + + counts = Counts() + needs_header: list[str] = [] + rc = 0 + + for target in args.paths: + if process_path(target, args.forced_type, ignored_types, args.recursive, counts, args.dry_run, needs_header) != 0: + rc = 1 + + if needs_header: + print(f"copyrighter: {SYM['summary']} files needing a copyright header:") + for entry in needs_header: + print(f" - {entry}") + if args.dry_run: + rc = 1 + + print() + print(" Run the script without -n|--dry-run|--check to actually apply changes.") + print() + print( + f"copyrighter: {SYM['summary']} added {counts.added}, updated {counts.updated}, " + f"unchanged {counts.unchanged}, skipped {counts.skipped}" + ) + return rc + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/tests/__init__.py b/bin/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/bin/tests/test_copyrighter.py b/bin/tests/test_copyrighter.py new file mode 100644 index 00000000..512386f9 --- /dev/null +++ b/bin/tests/test_copyrighter.py @@ -0,0 +1,524 @@ +# ----------------------------------------------------------------------------- +# (C) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +# ----------------------------------------------------------------------------- +""" +Unit tests for copyrighter +""" + +import os +import stat +from pathlib import Path + +import pytest + +from ..copyrighter import ( + BORDER, + MARKER, + Counts, + atomic_write, + collect_files, + detect_type, + generate_header, + main, + process_file, + process_path, + split_lines, +) + +# ------------------------------------------------------------------------------ +# generate_header +# ------------------------------------------------------------------------------ + + +def test_generate_header_python_hash_style(): + lines = generate_header("python") + assert lines[0] == f"# {BORDER}" + assert lines[1] == "# (C) Crown copyright Met Office. All rights reserved." + assert lines[4] == f"# {BORDER}" + assert len(lines) == 5 + + +def test_generate_header_fortran77_five_space_separator(): + lines = generate_header("fortran77") + assert lines[0] == f"C {BORDER}" + assert lines[1] == "C (C) Crown copyright Met Office. All rights reserved." + # the marker must be in column 1, not indented + assert lines[1].startswith("C") + + +def test_generate_header_fortran_free_form_uses_bang(): + lines = generate_header("fortran") + assert lines[0] == f"! {BORDER}" + + +def test_generate_header_markdown_wraps_html_comment(): + lines = generate_header("markdown") + assert lines[0] == "" + assert lines[1] == BORDER # no prefix, no separator + + +def test_generate_header_c_wraps_block_comment(): + lines = generate_header("c") + assert lines[0] == "/*" + assert lines[-1] == " */" + assert lines[1] == f" * {BORDER}" + + +def test_generate_header_rst_directive_syntax(): + lines = generate_header("rst") + assert lines[0] == f".. {BORDER}" + assert lines[-1] == f" {BORDER}" + + +def test_generate_header_json_uses_jsonc_style(): + lines = generate_header("json") + assert lines[0] == f"// {BORDER}" + + +def test_generate_header_unknown_type_raises(): + with pytest.raises(KeyError): + generate_header("not-a-real-type") + + +# ------------------------------------------------------------------------------ +# detect_type +# ------------------------------------------------------------------------------ + + +@pytest.mark.parametrize( + "name,expected", + [ + ("foo.py", "python"), + ("FOO.PY", "python"), # case-insensitive + ("foo.sh", "shell"), + ("foo.bash", "shell"), + ("foo.ksh", "shell"), + ("foo.cpp", "cpp"), + ("foo.hpp", "cpp"), + ("foo.c", "c"), + ("foo.h", "c"), + ("foo.f90", "fortran"), + ("foo.f95", "fortran"), + ("foo.f", "fortran77"), + ("foo.f77", "fortran77"), + ("foo.for", "fortran77"), + ("foo.go", "go"), + ("foo.java", "java"), + ("foo.js", "js"), + ("foo.mjs", "js"), + ("foo.json", "json"), + ("foo.md", "markdown"), + ("foo.pl", "perl"), + ("foo.pm", "perl"), + ("foo.ps1", "powershell"), + ("foo.ps", "postscript"), + ("foo.eps", "postscript"), + ("foo.r", "r"), + ("foo.rb", "ruby"), + ("foo.rs", "rust"), + ("foo.rst", "rst"), + ("foo.tex", "latex"), + ("foo.toml", "toml"), + ("foo.yaml", "yaml"), + ("foo.yml", "yaml"), + ("foo.awk", "awk"), + ], +) +def test_detect_type_by_extension(name, expected): + assert detect_type(name, "") == expected + + +@pytest.mark.parametrize( + "first_line,expected", + [ + ("#!/usr/bin/env python3", "python"), + ("#!/usr/bin/perl", "perl"), + ("#!/usr/bin/env ruby", "ruby"), + ("#!/bin/bash", "shell"), + ("#!/bin/sh", "shell"), + ("#!/usr/bin/ksh", "shell"), + ("not a shebang", ""), + ("#!/bin/strange-interpreter", ""), + ], +) +def test_detect_type_falls_back_to_shebang(first_line, expected): + assert detect_type("noextension", first_line) == expected + + +def test_detect_type_extension_wins_over_shebang(): + assert detect_type("foo.py", "#!/bin/bash") == "python" + + +def test_detect_type_hidden_dotfile_has_no_extension_match(): + # ".bashrc" splits to ext "bashrc", which is not a known extension + assert detect_type(".bashrc", "") == "" + assert detect_type(".bashrc", "#!/bin/bash") == "shell" + + +# ------------------------------------------------------------------------------ +# split_lines +# ------------------------------------------------------------------------------ + + +@pytest.mark.parametrize( + "text,expected", + [ + ("", []), + ("a\nb\n", ["a", "b"]), + ("a\nb", ["a", "b"]), + ("\n", [""]), + ("a", ["a"]), + ("a\n\nb\n", ["a", "", "b"]), + ], +) +def test_split_lines(text, expected): + assert split_lines(text) == expected + + +# ------------------------------------------------------------------------------ +# collect_files +# ------------------------------------------------------------------------------ + + +def test_collect_files_excludes_vcs_dirs_and_symlinks(tmp_path): + (tmp_path / ".git").mkdir() + (tmp_path / ".git" / "config").write_text("junk") + (tmp_path / ".svn").mkdir() + (tmp_path / ".svn" / "entries").write_text("junk") + (tmp_path / "sub").mkdir() + (tmp_path / "sub" / "a.py").write_text("print(1)\n") + (tmp_path / "top.py").write_text("print(1)\n") + real = tmp_path / "real.py" + real.write_text("print(1)\n") + (tmp_path / "link.py").symlink_to(real) + + non_recursive = collect_files(tmp_path, recursive=False) + assert sorted(p.name for p in non_recursive) == ["real.py", "top.py"] + + recursive = collect_files(tmp_path, recursive=True) + names = sorted(str(p.relative_to(tmp_path)) for p in recursive) + assert names == ["real.py", "sub/a.py", "top.py"] + + +def test_collect_files_non_recursive_ignores_subdirs(tmp_path): + (tmp_path / "sub").mkdir() + (tmp_path / "sub" / "a.py").write_text("print(1)\n") + (tmp_path / "top.py").write_text("print(1)\n") + + result = collect_files(tmp_path, recursive=False) + assert [p.name for p in result] == ["top.py"] + + +# ------------------------------------------------------------------------------ +# atomic_write +# ------------------------------------------------------------------------------ + + +def test_atomic_write_preserves_permissions_and_content(tmp_path): + target = tmp_path / "f.py" + target.write_text("old content\n") + target.chmod(0o640) + + atomic_write(target, b"new content\n") + + assert target.read_bytes() == b"new content\n" + assert stat.S_IMODE(target.stat().st_mode) == 0o640 + + +def test_atomic_write_no_stray_tempfile_left_behind(tmp_path): + target = tmp_path / "f.py" + target.write_text("old\n") + atomic_write(target, b"new\n") + assert list(tmp_path.iterdir()) == [target] + + +# ------------------------------------------------------------------------------ +# process_file +# ------------------------------------------------------------------------------ + + +def _run(path, forced_type=None, ignored_types=None, dry_run=False): + counts = Counts() + needs_header: list[str] = [] + process_file(path, forced_type, ignored_types or set(), counts, dry_run, needs_header) + return counts, needs_header + + +def test_process_file_adds_header_to_new_file(tmp_path): + f = tmp_path / "a.py" + f.write_text("print(1)\n") + + counts, needs_header = _run(f) + + assert counts.added == 1 + assert needs_header == [f"{f} (Added)"] + content = f.read_text() + assert MARKER in content + assert content.endswith("print(1)\n") + + +def test_process_file_preserves_shebang_before_header(tmp_path): + f = tmp_path / "a.py" + f.write_text("#!/usr/bin/env python3\nprint(1)\n") + + _run(f) + + lines = f.read_text().splitlines() + assert lines[0] == "#!/usr/bin/env python3" + assert MARKER in lines[2] + + +def test_process_file_preserves_postscript_magic_line(tmp_path): + f = tmp_path / "a.ps" + f.write_text("%!PS-Adobe-3.0\nshowpage\n") + + _run(f) + + lines = f.read_text().splitlines() + assert lines[0] == "%!PS-Adobe-3.0" + assert MARKER in lines[2] + + +def test_process_file_idempotent_second_run_is_unchanged(tmp_path): + f = tmp_path / "a.py" + f.write_text("print(1)\n") + + _run(f) + before = f.read_bytes() + counts, needs_header = _run(f) + + assert counts.unchanged == 1 + assert counts.added == 0 + assert needs_header == [] + assert f.read_bytes() == before + + +def test_process_file_updates_stale_header(tmp_path): + f = tmp_path / "a.py" + f.write_text( + "# -----\n" + "# (C) Crown copyright Met Office 2020. All rights reserved.\n" + "# The file LICENCE, distributed with this code, contains details of the terms\n" + "# under which the code may be used.\n" + "# -----\n" + "\n" + "print(1)\n" + ) + + counts, needs_header = _run(f) + + assert counts.updated == 1 + assert needs_header == [f"{f} (Updated)"] + content = f.read_text() + assert content.count(MARKER) == 1 + assert f"# {BORDER}" in content + + +def test_process_file_collapses_duplicate_headers(tmp_path): + header = "\n".join(generate_header("python")) + "\n" + f = tmp_path / "a.py" + f.write_text(header + header + "print(1)\n") + + counts, _ = _run(f) + + assert counts.updated == 1 + content = f.read_text() + assert content.count(MARKER) == 1 + + +def test_process_file_skips_ignored_type(tmp_path): + f = tmp_path / "a.py" + f.write_text("print(1)\n") + + counts, needs_header = _run(f, ignored_types={"python"}) + + assert counts.skipped == 1 + assert needs_header == [] + assert f.read_text() == "print(1)\n" + + +def test_process_file_skips_binary_file(tmp_path): + f = tmp_path / "a.py" + f.write_bytes(b"\x00\x01binary-data") + + counts, _ = _run(f) + + assert counts.skipped == 1 + assert f.read_bytes() == b"\x00\x01binary-data" + + +def test_process_file_skips_undetectable_type(tmp_path): + f = tmp_path / "noextension" + f.write_text("just some text\n") + + counts, _ = _run(f) + + assert counts.skipped == 1 + + +def test_process_file_skips_unreadable_file(tmp_path): + f = tmp_path / "a.py" + f.write_text("print(1)\n") + f.chmod(0o000) + try: + counts, _ = _run(f) + assert counts.skipped == 1 + finally: + f.chmod(0o644) + + +def test_process_file_forced_type_overrides_detection(tmp_path): + f = tmp_path / "a.weird" + f.write_text("puts 1\n") + + _run(f, forced_type="ruby") + + assert f.read_text().splitlines()[0] == "# " + BORDER + + +def test_process_file_json_prints_jsonc_warning(tmp_path, capsys): + f = tmp_path / "a.json" + f.write_text("{}\n") + + _run(f) + + captured = capsys.readouterr() + assert "JSONC-style" in captured.err + assert f.read_text().splitlines()[0] == "// " + BORDER + + +def test_process_file_dry_run_does_not_modify_file(tmp_path): + f = tmp_path / "a.py" + original = "print(1)\n" + f.write_text(original) + + counts, needs_header = _run(f, dry_run=True) + + assert counts.added == 1 + assert needs_header == [f"{f} (Added)"] + assert f.read_text() == original + + +# ------------------------------------------------------------------------------ +# process_path +# ------------------------------------------------------------------------------ + + +def test_process_path_missing_target_returns_error(tmp_path, capsys): + rc = process_path( + str(tmp_path / "missing"), None, set(), False, Counts(), False, [] + ) + assert rc == 1 + assert "does not exist" in capsys.readouterr().err + + +def test_process_path_file_delegates_to_process_file(tmp_path): + f = tmp_path / "a.py" + f.write_text("print(1)\n") + counts = Counts() + + rc = process_path(str(f), None, set(), False, counts, False, []) + + assert rc == 0 + assert counts.added == 1 + + +def test_process_path_directory_recursive_vs_non_recursive(tmp_path): + (tmp_path / "sub").mkdir() + (tmp_path / "sub" / "b.py").write_text("print(2)\n") + (tmp_path / "a.py").write_text("print(1)\n") + + counts = Counts() + process_path(str(tmp_path), None, set(), False, counts, False, []) + assert counts.added == 1 # only a.py + + counts = Counts() + process_path(str(tmp_path), None, set(), True, counts, False, []) + assert counts.added == 1 # only sub/b.py remains un-headered + + +# ------------------------------------------------------------------------------ +# main (CLI) +# ------------------------------------------------------------------------------ + + +def test_main_dry_run_exits_nonzero_when_changes_pending(tmp_path, capsys): + f = tmp_path / "a.py" + f.write_text("print(1)\n") + + rc = main(["--dry-run", str(f)]) + + assert rc == 1 + assert f.read_text() == "print(1)\n" + + +def test_main_exits_zero_when_nothing_to_do(tmp_path): + f = tmp_path / "a.py" + f.write_text("print(1)\n") + main([str(f)]) + + rc = main(["--dry-run", str(f)]) + + assert rc == 0 + + +def test_main_nonexistent_path_exits_nonzero(tmp_path, capsys): + rc = main([str(tmp_path / "missing.py")]) + assert rc == 1 + assert "does not exist" in capsys.readouterr().err + + +def test_main_ignore_option_comma_separated_and_repeatable(tmp_path): + py = tmp_path / "a.py" + rb = tmp_path / "b.rb" + sh = tmp_path / "c.sh" + py.write_text("print(1)\n") + rb.write_text("puts 1\n") + sh.write_text("echo hi\n") + + main(["--recursive", "--ignore", "python,ruby", str(tmp_path)]) + + assert MARKER not in py.read_text() + assert MARKER not in rb.read_text() + assert MARKER in sh.read_text() + + +def test_main_ignore_option_rejects_unknown_type(tmp_path, capsys): + f = tmp_path / "a.py" + f.write_text("print(1)\n") + + with pytest.raises(SystemExit) as exc_info: + main(["--ignore", "not-a-type", str(f)]) + + assert exc_info.value.code == 2 + assert "unknown type" in capsys.readouterr().err + + +def test_main_forced_type_via_cli(tmp_path): + f = tmp_path / "a.weird" + f.write_text("puts 1\n") + + main(["--type", "ruby", str(f)]) + + assert f.read_text().splitlines()[0] == "# " + BORDER + + +def test_main_recursive_flag_descends_into_subdirectories(tmp_path): + (tmp_path / "sub").mkdir() + nested = tmp_path / "sub" / "b.py" + nested.write_text("print(2)\n") + + main(["--recursive", str(tmp_path)]) + + assert MARKER in nested.read_text() + + +def test_script_is_executable_and_has_shebang(): + script = Path(__file__).resolve().parents[1] / "copyrighter.py" + assert script.exists() + assert os.access(script, os.X_OK) + with open(script) as fh: + assert fh.readline().startswith("#!") diff --git a/sbin/copyrighter b/sbin/copyrighter deleted file mode 100755 index 343d195a..00000000 --- a/sbin/copyrighter +++ /dev/null @@ -1,455 +0,0 @@ -#!/usr/bin/env bash -# ----------------------------------------------------------------------------- -# (C) Crown copyright Met Office. All rights reserved. -# The file LICENCE, distributed with this code, contains details of the terms -# under which the code may be used. -# ----------------------------------------------------------------------------- - -# copyrighter - insert or update the Met Office Crown copyright header in a -# file, or in every file under a directory. -# -# Usage: copyrighter [-t TYPE] [-i TYPE] [-r] [-n] [-h] [path...] - -set -euo pipefail - -readonly BORDER="-----------------------------------------------------------------------------" -readonly MARKER="Crown copyright Met Office" -readonly SUPPORTED_TYPES="python fortran cpp c shell rst markdown perl yaml json toml rust ruby go js latex awk java powershell r postscript" -readonly VCS_DIRS=(.git .svn) - -# status symbols used to decorate every "copyrighter:" message -readonly SYM_ADDED="+" -readonly SYM_UPDATED="~" -readonly SYM_UNCHANGED="=" -readonly SYM_WARN="!" -readonly SYM_INFO="i" -readonly SYM_ERROR="✗" -readonly SYM_SUMMARY="»" - -FORCED_TYPE="" -IGNORED_TYPES=() -RECURSIVE=0 -DRY_RUN=0 -ADDED_COUNT=0 -UPDATED_COUNT=0 -UNCHANGED_COUNT=0 -SKIPPED_COUNT=0 -HEADER_LINES=() -NEEDS_HEADER_FILES=() - -# prefer fd when available for directory traversal, falling back to find -# (ripgrep is not used for binary-file detection, see is_text_file) -HAVE_FD=0 -if command -v fd >/dev/null 2>&1; then HAVE_FD=1; fi - -usage() { - cat < [path...] - -Insert a Met Office Crown copyright header at the top of a file (after any -shebang line), or into every file under a directory. If a copyrighter header -is already present it is updated in place rather than duplicated. - -Options: - -t, --type TYPE Force the file type instead of auto-detecting it. - One of: ${SUPPORTED_TYPES} - -i, --ignore TYPE Skip files whose (auto-detected or forced) type matches. - Accepts a comma-separated list and may be given more - than once; repeats and lists accumulate. - -r, --recursive, --full - When a directory is given, descend into every nested - subdirectory (no depth limit). Without this, only the - files directly inside the given directory are processed. - -n, --dry-run, --check - Report what would change without modifying any file. - -h, --help Show this help message. - -File types are auto-detected from the file extension, falling back to the -shebang line for extension-less scripts. JSON has no native comment syntax, -so headers are inserted as "//" line comments (JSONC-style); a warning is -printed whenever this happens. - -Exits non-zero if any file needed a header added or updated, or if a path -could not be processed - useful for CI checks with --dry-run. -EOF -} - -# detect_type FILE - auto-detect a file's type from its extension, falling -# back to the shebang line when the extension is missing or unrecognised. -detect_type() { - local file="$1" ext first_line - - ext="${file##*.}" - [[ "$file" == *.* && "$ext" != "$file" ]] || ext="" - ext="${ext,,}" - - case "$ext" in - awk) echo "awk"; return ;; - bash|ksh|sh) echo "shell"; return ;; - c++|cc|cpp|cxx|h++|hh|hpp|hxx) echo "cpp"; return ;; - c|h) echo "c"; return ;; - f|f77|f90|f95|f03|f08|for) echo "fortran"; return ;; - go) echo "go"; return ;; - java) echo "java"; return ;; - js|cjs|mjs) echo "js"; return ;; - json) echo "json"; return ;; - markdown|md) echo "markdown"; return ;; - pl|pm) echo "perl"; return ;; - ps1|psm1|psd1) echo "powershell"; return ;; - ps|eps) echo "postscript"; return ;; - py) echo "python"; return ;; - r) echo "r"; return ;; - rb) echo "ruby"; return ;; - rs) echo "rust"; return ;; - rst) echo "rst"; return ;; - tex) echo "latex"; return ;; - toml) echo "toml"; return ;; - yaml|yml) echo "yaml"; return ;; - esac - - first_line=$(head -n 1 -- "$file" 2>/dev/null || true) - case "$first_line" in - '#!'*python*) echo "python"; return ;; - '#!'*perl*) echo "perl"; return ;; - '#!'*ruby*) echo "ruby"; return ;; - '#!'*bash|'#!'*/sh|'#!'*ksh) echo "shell"; return ;; - esac - - echo "" -} - -# generate_header TYPE - populate the HEADER_LINES array with the -# comment-style-appropriate copyright block for TYPE. -generate_header() { - local type="$1" - local c_style=0 - - # Determine the comment character/style based on file type - case "$type" in - python|shell|yaml|perl|toml|ruby|awk|powershell|r) local p="#" ;; - fortran) local p="!" ;; - cpp|json|java|rust|go|js) local p="//" ;; - latex|postscript) local p="%" ;; - c) local p=" *" ; c_style=1 ;; - rst) local p=" " ;; # Left-hand indent for body lines - markdown) local p="" ;; # No prefix inside HTML comments - *) echo "copyrighter: ${SYM_ERROR} internal error: unknown type '${type}'" >&2 - return 1 ;; - esac - - # Construct the core body of uniform copyright block - HEADER_LINES=( - "${p}${p:+ }${BORDER}" - "${p}${p:+ }(C) Crown copyright Met Office. All rights reserved." - "${p}${p:+ }The file LICENCE, distributed with this code, contains details of the terms" - "${p}${p:+ }under which the code may be used." - "${p}${p:+ }${BORDER}" - ) - - # Add language-specific wrapping for block-comment formats - if [ "$type" = "rst" ]; then - # Replace the first and last lines with the correct RST directive syntax - HEADER_LINES[0]=".. ${BORDER}" - HEADER_LINES[4]=" ${BORDER}" - elif [ "$type" = "markdown" ]; then - HEADER_LINES=("") - elif [ $c_style -eq 1 ]; then - HEADER_LINES=("/*" "${HEADER_LINES[@]}" " */") - fi -} - -# is_text_file FILE - true if FILE looks like text, false if binary -# -# always uses grep -I: unlike find/rg, ripgrep has no quiet flag that -# reliably reports binary-ness via its exit code (an empty pattern still -# "matches" binary files up to their first NUL byte), so using it here -# risks corrupting binaries by writing a text header into them -is_text_file() { - grep -Iq '' -- "$1" 2>/dev/null -} - -# is_type_ignored TYPE - true if TYPE was named via -i/--ignore -is_type_ignored() { - local type="$1" ignored - for ignored in "${IGNORED_TYPES[@]}"; do - [[ "$ignored" == "$type" ]] && return 0 - done - return 1 -} - -# collect_files DIR RECURSIVE - print a NUL-separated, sorted list of regular -# files under DIR, using fd when available and falling back to find; version -# control directories (VCS_DIRS) are always excluded from the walk -collect_files() { - local dir="$1" recursive="$2" - - if ((HAVE_FD)); then - local fd_opts=(--type f --hidden --no-ignore --absolute-path --print0) - local vcs_dir - for vcs_dir in "${VCS_DIRS[@]}"; do fd_opts+=(--exclude "$vcs_dir"); done - ((recursive == 0)) && fd_opts+=(--max-depth 1) - fd "${fd_opts[@]}" . "$dir" | sort -z - else - local prune_expr=() vcs_dir maxdepth_opts=() - for vcs_dir in "${VCS_DIRS[@]}"; do - ((${#prune_expr[@]} > 0)) && prune_expr+=(-o) - prune_expr+=(-name "$vcs_dir") - done - ((recursive == 0)) && maxdepth_opts=(-maxdepth 1) - find "$dir" "${maxdepth_opts[@]}" \( "${prune_expr[@]}" \) -prune -o -type f -print0 | sort -z - fi -} - -# Process_file FILE FORCED_TYPE - insert or update the copyright header in FILE. -process_file() { - local file="$1" forced_type="$2" type - - if ! is_text_file "$file"; then - echo "copyrighter: ${SYM_WARN} skipping '${file}' (binary file)" >&2 - ((++SKIPPED_COUNT)) - return 0 - fi - - type="${forced_type:-$(detect_type "$file")}" - if [[ -z "$type" ]]; then - echo "copyrighter: ${SYM_WARN} skipping '${file}' (unable to determine file type, use --type)" >&2 - ((++SKIPPED_COUNT)) - return 0 - fi - - if is_type_ignored "$type"; then - echo "copyrighter: ${SYM_INFO} skipping '${file}' (ignored type '${type}')" >&2 - ((++SKIPPED_COUNT)) - return 0 - fi - - if [[ "$type" == "json" ]]; then - echo "copyrighter: ${SYM_INFO} JSON has no comment syntax; '${file}' will become JSONC-style" >&2 - fi - - generate_header "$type" - local header_count=${#HEADER_LINES[@]} - - local lines=() - mapfile -t lines < "$file" - local nlines=${#lines[@]} - - local offset=0 - if [[ "${lines[0]:-}" =~ ^#! ]]; then - offset=1 - elif [[ "$type" == "postscript" && "${lines[0]:-}" =~ ^%! ]]; then - offset=1 # preserve the required %!PS-Adobe magic line - fi - - # find the relative position of the marker line within our own generated - # template (not file content), used to anchor the regex built below - local marker_rel=-1 i - for ((i = 0; i < header_count; i++)); do - if [[ "${HEADER_LINES[$i]}" == *"$MARKER"* ]]; then - marker_rel=$i - break - fi - done - - # match the marker line by prefix + a fixed regex body, tolerating an - # optional copyright year (e.g. "Met Office 2026."), so a pre-existing - # header isn't mistaken for new content and duplicated - local marker_prefix marker_escaped_prefix marker_regex - marker_prefix="${HEADER_LINES[$marker_rel]%%(C) Crown copyright*}" - marker_escaped_prefix=$(printf '%s' "$marker_prefix" | sed -e 's/[\\.^$*+?(){}|]/\\&/g') - marker_regex="^${marker_escaped_prefix}\(C\) Crown copyright Met Office( [0-9]{4})?\. All rights reserved\.$" - - local search_limit=$((offset + 100)) - ((search_limit > nlines)) && search_limit=$nlines - - # scan for every header-like block near the top of the file (not just the - # first) so that stale duplicates, e.g. left behind by an older version of - # this script, are all collapsed into a single canonical header - local block_starts=() block_ends=() - i=$offset - while ((i < search_limit)); do - if [[ "${lines[$i]}" =~ $marker_regex ]]; then - local candidate_start=$((i - marker_rel)) - local candidate_end=$((candidate_start + header_count - 1)) - if ((candidate_start >= offset && candidate_end < nlines)); then - block_starts+=("$candidate_start") - block_ends+=("$candidate_end") - i=$((candidate_end + 1)) - continue - fi - fi - ((++i)) - done - - local action="Added" - if ((${#block_starts[@]} == 1)); then - action="Updated" - local k matches=1 - for ((k = 0; k < header_count; k++)); do - if [[ "${lines[$((block_starts[0] + k))]}" != "${HEADER_LINES[$k]}" ]]; then - matches=0 - break - fi - done - ((matches)) && action="Unchanged" - elif ((${#block_starts[@]} > 1)); then - action="Updated" - fi - - if [[ "$action" == "Unchanged" ]]; then - echo "copyrighter: ${SYM_UNCHANGED} Unchanged ${type} header in '${file}' (already up to date)" - ((++UNCHANGED_COUNT)) - return 0 - fi - - # rebuild the file's content with every detected header block removed - local tail=() idx=$offset block_idx=0 - while ((idx < nlines)); do - if ((block_idx < ${#block_starts[@]} && idx == block_starts[block_idx])); then - idx=$((block_ends[block_idx] + 1)) - ((++block_idx)) - continue - fi - tail+=("${lines[$idx]}") - ((++idx)) - done - while ((${#tail[@]} > 0)) && [[ -z "${tail[0]}" ]]; do - tail=("${tail[@]:1}") - done - - local new_lines=() - ((offset > 0)) && new_lines+=("${lines[@]:0:offset}") - new_lines+=("${HEADER_LINES[@]}") - new_lines+=("") - ((${#tail[@]} > 0)) && new_lines+=("${tail[@]}") - - local symbol="$SYM_ADDED" - [[ "$action" == "Updated" ]] && symbol="$SYM_UPDATED" - NEEDS_HEADER_FILES+=("${file} (${action})") - if [[ "$action" == "Added" ]]; then - ((++ADDED_COUNT)) - else - ((++UPDATED_COUNT)) - fi - - if ((DRY_RUN)); then - echo "copyrighter: ${symbol} [dry-run] ${action} ${type} header in '${file}'" - return 0 - fi - - # Write the new header to a temporary file and replace the original file with it. - local perms tmpfile - perms=$(stat -c '%a' "$file" 2>/dev/null || echo "") - tmpfile="${file}.copyrighter.tmp.$$" - printf '%s\n' "${new_lines[@]}" > "$tmpfile" - [[ -n "$perms" ]] && chmod "$perms" "$tmpfile" - mv -- "$tmpfile" "$file" - - echo "copyrighter: ${symbol} ${action} ${type} header in '${file}'" -} - -# process_path PATH - dispatch a single CLI argument to a file or directory walk. -process_path() { - local target="$1" - - if [[ -f "$target" ]]; then - if [[ -n "$FORCED_TYPE" ]]; then - process_file "$target" "$FORCED_TYPE" - else - local detected - detected="$(detect_type "$target")" - if [[ -z "$detected" ]]; then - echo "copyrighter: ${SYM_WARN} unable to determine file type for '${target}', use --type" >&2 - ((++SKIPPED_COUNT)) - return 0 - fi - process_file "$target" "$detected" - fi - return 0 - fi - - if [[ -d "$target" ]]; then - local f - while IFS= read -r -d '' f; do - process_file "$f" "$FORCED_TYPE" - done < <(collect_files "$target" "$RECURSIVE") - return 0 - fi - - echo "copyrighter: ${SYM_ERROR} '${target}' does not exist" >&2 - return 1 -} - -main() { - local paths=() - - while (($# > 0)); do - case "$1" in - -t|--type) - [[ $# -ge 2 ]] || { echo "copyrighter: ${SYM_ERROR} --type requires an argument" >&2; exit 1; } - FORCED_TYPE="$2" - if [[ ! " ${SUPPORTED_TYPES} " == *" ${FORCED_TYPE} "* ]]; then - echo "copyrighter: ${SYM_ERROR} unknown type '${FORCED_TYPE}' (expected one of: ${SUPPORTED_TYPES})" >&2 - exit 1 - fi - shift 2 ;; - -i|--ignore) - [[ $# -ge 2 ]] || { echo "copyrighter: ${SYM_ERROR} --ignore requires an argument" >&2; exit 1; } - local ignore_arg ignore_list - IFS=',' read -r -a ignore_list <<< "$2" - for ignore_arg in "${ignore_list[@]}"; do - if [[ ! " ${SUPPORTED_TYPES} " == *" ${ignore_arg} "* ]]; then - echo "copyrighter: ${SYM_ERROR} unknown type '${ignore_arg}' (expected one of: ${SUPPORTED_TYPES})" >&2 - exit 1 - fi - IGNORED_TYPES+=("$ignore_arg") - done - shift 2 ;; - -r|--recursive|--full) - RECURSIVE=1 - shift ;; - -n|--dry-run|--check) - DRY_RUN=1 - shift ;; - -h|--help) - usage - exit 0 ;; - --) - shift - while (($# > 0)); do paths+=("$1"); shift; done ;; - -*) - echo "copyrighter: ${SYM_ERROR} unknown option '$1'" >&2 - usage >&2 - exit 1 ;; - *) - paths+=("$1") - shift ;; - esac - done - - if ((${#paths[@]} == 0)); then - usage >&2 - exit 1 - fi - - local rc=0 p - for p in "${paths[@]}"; do - process_path "$p" || rc=1 - done - - if ((${#NEEDS_HEADER_FILES[@]} > 0)); then - echo "copyrighter: ${SYM_SUMMARY} files needing a copyright header:" - local entry - for entry in "${NEEDS_HEADER_FILES[@]}"; do - echo " - ${entry}" - done - if ((DRY_RUN)); then rc=1; fi - fi - echo "" - echo " Run the script without -n|--dry-run|--check to actually apply changes." - echo "" - echo "copyrighter: ${SYM_SUMMARY} added ${ADDED_COUNT}, updated ${UPDATED_COUNT}, unchanged ${UNCHANGED_COUNT}, skipped ${SKIPPED_COUNT}" - exit "$rc" -} - -main "$@" From 520f3582947ec6f95d39008b0206379894767a08 Mon Sep 17 00:00:00 2001 From: yaswant <2984440+yaswant@users.noreply.github.com> Date: Sat, 10 Oct 2026 02:42:12 +0100 Subject: [PATCH 6/6] Apply ruff --- bin/copyrighter.py | 166 +++++++++++++++++++++++++++------- bin/tests/test_copyrighter.py | 4 +- 2 files changed, 134 insertions(+), 36 deletions(-) diff --git a/bin/copyrighter.py b/bin/copyrighter.py index a967ca94..90bb2b16 100755 --- a/bin/copyrighter.py +++ b/bin/copyrighter.py @@ -29,20 +29,41 @@ EXTENSION_TO_TYPE: dict[str, str] = { "awk": "awk", - "bash": "shell", "ksh": "shell", "sh": "shell", - "c++": "cpp", "cc": "cpp", "cpp": "cpp", "cxx": "cpp", - "h++": "cpp", "hh": "cpp", "hpp": "cpp", "hxx": "cpp", - "c": "c", "h": "c", - "f90": "fortran", "f95": "fortran", "f03": "fortran", "f08": "fortran", - "f": "fortran77", "f77": "fortran77", "for": "fortran77", + "bash": "shell", + "ksh": "shell", + "sh": "shell", + "c++": "cpp", + "cc": "cpp", + "cpp": "cpp", + "cxx": "cpp", + "h++": "cpp", + "hh": "cpp", + "hpp": "cpp", + "hxx": "cpp", + "c": "c", + "h": "c", + "f90": "fortran", + "f95": "fortran", + "f03": "fortran", + "f08": "fortran", + "f": "fortran77", + "f77": "fortran77", + "for": "fortran77", "go": "go", "java": "java", - "js": "js", "cjs": "js", "mjs": "js", + "js": "js", + "cjs": "js", + "mjs": "js", "json": "json", - "markdown": "markdown", "md": "markdown", - "pl": "perl", "pm": "perl", - "ps1": "powershell", "psm1": "powershell", "psd1": "powershell", - "ps": "postscript", "eps": "postscript", + "markdown": "markdown", + "md": "markdown", + "pl": "perl", + "pm": "perl", + "ps1": "powershell", + "psm1": "powershell", + "psd1": "powershell", + "ps": "postscript", + "eps": "postscript", "py": "python", "r": "r", "rb": "ruby", @@ -50,17 +71,31 @@ "rst": "rst", "tex": "latex", "toml": "toml", - "yaml": "yaml", "yml": "yaml", + "yaml": "yaml", + "yml": "yaml", } # comment-prefix per type; "c", "rst" and "markdown" get extra wrapping below COMMENT_PREFIX: dict[str, str] = { - "python": "#", "shell": "#", "yaml": "#", "perl": "#", "toml": "#", - "ruby": "#", "awk": "#", "powershell": "#", "r": "#", + "python": "#", + "shell": "#", + "yaml": "#", + "perl": "#", + "toml": "#", + "ruby": "#", + "awk": "#", + "powershell": "#", + "r": "#", "fortran": "!", "fortran77": "C", # fixed-form F77: marker must be in column 1 - "cpp": "//", "json": "//", "java": "//", "rust": "//", "go": "//", "js": "//", - "latex": "%", "postscript": "%", + "cpp": "//", + "json": "//", + "java": "//", + "rust": "//", + "go": "//", + "js": "//", + "latex": "%", + "postscript": "%", "c": " *", "rst": " ", "markdown": "", @@ -69,8 +104,13 @@ SUPPORTED_TYPES = tuple(sorted(set(COMMENT_PREFIX))) SYM = { - "added": "+", "updated": "~", "unchanged": "=", - "warn": "!", "info": "i", "error": "\u2717", "summary": "\u00bb", + "added": "+", + "updated": "~", + "unchanged": "=", + "warn": "!", + "info": "i", + "error": "\u2717", + "summary": "\u00bb", } @@ -121,7 +161,7 @@ def _shebang_type(first_line: str) -> str: return "perl" if "ruby" in first_line: return "ruby" - if first_line.endswith("bash") or first_line.endswith("/sh") or first_line.endswith("ksh"): + if first_line.endswith(("bash", "/sh", "ksh")): return "shell" return "" @@ -173,7 +213,9 @@ def atomic_write(path: Path, data: bytes) -> None: permission bits, using a securely-created (O_EXCL) temp file to avoid a symlink race on the temporary path.""" mode = stat.S_IMODE(path.stat().st_mode) - fd, tmp_name = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.copyrighter.") + fd, tmp_name = tempfile.mkstemp( + dir=path.parent, prefix=f".{path.name}.copyrighter." + ) try: with os.fdopen(fd, "wb") as fh: fh.write(data) @@ -198,12 +240,18 @@ def process_file( try: raw = path.read_bytes() except OSError as exc: - print(f"copyrighter: {SYM['warn']} skipping '{path}' (cannot read: {exc.strerror})", file=sys.stderr) + print( + f"copyrighter: {SYM['warn']} skipping '{path}' (cannot read: {exc.strerror})", + file=sys.stderr, + ) counts.skipped += 1 return if b"\x00" in raw: - print(f"copyrighter: {SYM['warn']} skipping '{path}' (binary file)", file=sys.stderr) + print( + f"copyrighter: {SYM['warn']} skipping '{path}' (binary file)", + file=sys.stderr, + ) counts.skipped += 1 return @@ -213,17 +261,26 @@ def process_file( file_type = forced_type or detect_type(str(path), first_line) if not file_type: - print(f"copyrighter: {SYM['warn']} skipping '{path}' (unable to determine file type, use --type)", file=sys.stderr) + print( + f"copyrighter: {SYM['warn']} skipping '{path}' (unable to determine file type, use --type)", + file=sys.stderr, + ) counts.skipped += 1 return if file_type in ignored_types: - print(f"copyrighter: {SYM['info']} skipping '{path}' (ignored type '{file_type}')", file=sys.stderr) + print( + f"copyrighter: {SYM['info']} skipping '{path}' (ignored type '{file_type}')", + file=sys.stderr, + ) counts.skipped += 1 return if file_type == "json": - print(f"copyrighter: {SYM['info']} JSON has no comment syntax; '{path}' will become JSONC-style", file=sys.stderr) + print( + f"copyrighter: {SYM['info']} JSON has no comment syntax; '{path}' will become JSONC-style", + file=sys.stderr, + ) header_lines = generate_header(file_type) header_count = len(header_lines) @@ -238,7 +295,8 @@ def process_file( marker_rel = next((i for i, hl in enumerate(header_lines) if MARKER in hl), -1) marker_prefix = header_lines[marker_rel].split("(C) Crown copyright", 1)[0] marker_re = re.compile( - re.escape(marker_prefix) + r"\(C\) Crown copyright Met Office(?: \d{4})?\. All rights reserved\." + re.escape(marker_prefix) + + r"\(C\) Crown copyright Met Office(?: \d{4})?\. All rights reserved\." ) search_limit = min(offset + 100, nlines) @@ -262,12 +320,18 @@ def process_file( action = "Added" if len(block_starts) == 1: start = block_starts[0] - action = "Unchanged" if lines[start:start + header_count] == header_lines else "Updated" + action = ( + "Unchanged" + if lines[start : start + header_count] == header_lines + else "Updated" + ) elif len(block_starts) > 1: action = "Updated" if action == "Unchanged": - print(f"copyrighter: {SYM['unchanged']} Unchanged {file_type} header in '{path}' (already up to date)") + print( + f"copyrighter: {SYM['unchanged']} Unchanged {file_type} header in '{path}' (already up to date)" + ) counts.unchanged += 1 return @@ -295,7 +359,9 @@ def process_file( counts.updated += 1 if dry_run: - print(f"copyrighter: {symbol} [dry-run] {action} {file_type} header in '{path}'") + print( + f"copyrighter: {symbol} [dry-run] {action} {file_type} header in '{path}'" + ) return new_text = "\n".join(new_lines) + "\n" @@ -349,20 +415,37 @@ def build_parser() -> argparse.ArgumentParser: ) parser.add_argument("paths", nargs="+", metavar="file-or-directory") parser.add_argument( - "-t", "--type", dest="forced_type", choices=SUPPORTED_TYPES, metavar="TYPE", + "-t", + "--type", + dest="forced_type", + choices=SUPPORTED_TYPES, + metavar="TYPE", help="Force the file type instead of auto-detecting it.", ) parser.add_argument( - "-i", "--ignore", dest="ignore", action="append", default=[], metavar="TYPE", + "-i", + "--ignore", + dest="ignore", + action="append", + default=[], + metavar="TYPE", help="Skip files whose (auto-detected or forced) type matches. Accepts " "a comma-separated list and may be given more than once.", ) parser.add_argument( - "-r", "--recursive", "--full", dest="recursive", action="store_true", + "-r", + "--recursive", + "--full", + dest="recursive", + action="store_true", help="When a directory is given, descend into every nested subdirectory.", ) parser.add_argument( - "-n", "--dry-run", "--check", dest="dry_run", action="store_true", + "-n", + "--dry-run", + "--check", + dest="dry_run", + action="store_true", help="Report what would change without modifying any file.", ) return parser @@ -379,7 +462,9 @@ def main(argv: list[str] | None = None) -> int: if not item: continue if item not in SUPPORTED_TYPES: - parser.error(f"unknown type '{item}' (expected one of: {' '.join(SUPPORTED_TYPES)})") + parser.error( + f"unknown type '{item}' (expected one of: {' '.join(SUPPORTED_TYPES)})" + ) ignored_types.add(item) counts = Counts() @@ -387,7 +472,18 @@ def main(argv: list[str] | None = None) -> int: rc = 0 for target in args.paths: - if process_path(target, args.forced_type, ignored_types, args.recursive, counts, args.dry_run, needs_header) != 0: + if ( + process_path( + target, + args.forced_type, + ignored_types, + args.recursive, + counts, + args.dry_run, + needs_header, + ) + != 0 + ): rc = 1 if needs_header: diff --git a/bin/tests/test_copyrighter.py b/bin/tests/test_copyrighter.py index 512386f9..9e55ef99 100644 --- a/bin/tests/test_copyrighter.py +++ b/bin/tests/test_copyrighter.py @@ -243,7 +243,9 @@ def test_atomic_write_no_stray_tempfile_left_behind(tmp_path): def _run(path, forced_type=None, ignored_types=None, dry_run=False): counts = Counts() needs_header: list[str] = [] - process_file(path, forced_type, ignored_types or set(), counts, dry_run, needs_header) + process_file( + path, forced_type, ignored_types or set(), counts, dry_run, needs_header + ) return counts, needs_header