From 6904e7962f47024a3d11913bbc416bec781fa66b Mon Sep 17 00:00:00 2001 From: dayfine Date: Fri, 6 Nov 2026 22:51:14 -0800 Subject: [PATCH] Preview and apply same-line formatting only within Git-edited C++ lines --- devtools/format/INCREMENTAL.md | 91 ++++++++++++ devtools/format/format_changed.py | 222 ++++++++++++++++++++++++++++++ devtools/format/test_changed.py | 149 ++++++++++++++++++++ 3 files changed, 462 insertions(+) create mode 100644 devtools/format/INCREMENTAL.md create mode 100644 devtools/format/format_changed.py create mode 100644 devtools/format/test_changed.py diff --git a/devtools/format/INCREMENTAL.md b/devtools/format/INCREMENTAL.md new file mode 100644 index 0000000000..21d2e5bf0a --- /dev/null +++ b/devtools/format/INCREMENTAL.md @@ -0,0 +1,91 @@ +# Format edited lines only + +Use this mode for routine development. It avoids the broad reindentation and +known declaration-spacing changes in the full-file formatter preview. + +The tool needs Python 3, Uncrustify 0.83.0 and Clang (for a raw-token check). +It does not build or run SerialPrograms or connect a console. + +From your code worktree, preview first: + +```sh +python3 /path/to/tools/devtools/format/format_changed.py --repo . +``` + +While the tools PR is under review, the local checkout path is: + +```sh +python3 ~/Projects/wt/incremental-cpp-format/devtools/format/format_changed.py --repo . +``` + +Then apply the reviewed proposal: + +```sh +python3 ~/Projects/wt/incremental-cpp-format/devtools/format/format_changed.py --repo . --write +``` + +No directory change is necessary: `--repo ~/Projects/wt/bdsp-my-fix` selects +that checkout explicitly. The tool always uses the configuration beside its own +script, so the target branch need not merge the tools PR first. + +## Scope and behavior + +Default comparison is **HEAD against the working tree**, including staged and +unstaged edits. No commit is required. New files must first be tracked with +`git add`; untracked, deleted and renamed files are excluded. If a rename also +contains edits, commit the rename separately before using the default local mode. +Only `.h`, `.c`, `.cpp` and `.tpp` files in Common/SerialPrograms are considered. + +For a whole branch, compare with its merge base with a chosen parent: + +```sh +python3 /path/to/format_changed.py --repo . --base origin/main +``` + +Choose the actual parent for a stacked PR; do not compare a large story chain +with main when you only intend to format its top layer. The tool does not fetch. + +- Runs Uncrustify on each selected file to obtain context, then accepts only + same-line whitespace edits on lines Git marks added/modified. +- Keeps every original leading column, including manual declaration alignment, + and every original newline. Untouched lines remain byte-identical. +- Skips formatter hunks that change line count, preprocessor directives and + their continuations, and lines containing class/struct/union keywords. Those + skips keep block reflows and known declaration-brace changes out of the patch. +- Checks the candidate with Clang's raw lexer before writing: significant token + kind/spelling must match, including quoted/raw string contents. This does not + preprocess includes or prove every macro-invocation behavior equivalent; + review the displayed patch. Macro definitions are protected; invocations in + ordinary code are not universally identifiable without preprocessing. +- Verifies all candidates first and refuses a stale source snapshot. It never + stages files, commits, switches branches or changes the index. After --write, + review and stage the formatting yourself. + +This mode intentionally does **not** fix indentation, insert braces, expand +single-line control bodies, reflow blocks or assert complete style compliance. +Use existing lint/review for those. Skipped selected edits are reported on stderr; +stdout is the eligible patch. An unchanged result can mean the remaining +formatter suggestions were deliberately outside this mode's scope. + +`--check` performs the same preview without writing and returns 1 if eligible +formatting remains; 0 means no eligible changes, and 2 indicates a tool, lexer, +version or source-snapshot error. Preview alone returns 0 without writing. +There is no automatic hook or source-format CI gate in this PR. + +```sh +python3 /path/to/format_changed.py --repo . --check +``` + +Use `--uncrustify /path/to/uncrustify` / `--clang /path/to/clang` or environment +variables `UNCRUSTIFY` / `CLANG` for non-PATH installations. + +## Tests + +```sh +python3 -m unittest discover -s devtools/format -v +``` + +Tests call the real formatter and lexer, and use temporary Git repositories for +preview/check/write, staged-index preservation, explicit bases, unusual file +names, stale snapshots and refusal-before-write behavior. Application tests +remain native in their owning modules. diff --git a/devtools/format/format_changed.py b/devtools/format/format_changed.py new file mode 100644 index 0000000000..c39d90d6dd --- /dev/null +++ b/devtools/format/format_changed.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Preview whitespace formatting of edited C/C++ lines, from any working directory. + + python3 /path/to/tools/devtools/format/format_changed.py --repo /path/to/code + python3 devtools/format/format_changed.py --write + python3 devtools/format/format_changed.py --base origin/main --check + +Uses the config beside this tool, Uncrustify 0.83.0 and Clang's raw lexer. +Default scope is staged + unstaged tracked edits against HEAD; --base selects +its merge base with HEAD. Never stages, commits, or reformats untouched lines. +See INCREMENTAL.md for the intentionally narrow formatting scope. +""" + +import argparse +from collections import Counter +from dataclasses import dataclass +import difflib +import os +from pathlib import Path +import re +import subprocess +import sys + + +CONFIG = Path(__file__).resolve().parents[2] / '.uncrustify.cfg' +SUFFIXES = {'.h', '.c', '.cpp', '.tpp'} + + +@dataclass +class Selection: + """Changed lines tied to the exact bytes read when their Git diff was measured.""" + lines: set + source: bytes + + +@dataclass +class Plan: + """A verified candidate plus the original bytes needed to detect stale plans.""" + path: Path + before: bytes + after: bytes + skipped: dict + + +def git(repo, *args): + """Read Git state; never invoke shell expansion, external diff or textconv.""" + return subprocess.check_output(['git', '-C', str(repo), *args]) + + +def hunk_lines(diff): + """Return one-based added/modified working-tree line numbers, not deletion anchors.""" + lines = set() + for match in re.finditer(rb'^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@', diff, re.M): + start = int(match[1]) + count = int(match[2]) if match[2] is not None else 1 + lines.update(range(start, start + count)) + return lines + + +def selected_lines(repo, base): + """Select tracked additions/modifications; ignore renamed and untracked files.""" + command = ['diff', '--no-ext-diff', '--no-textconv', '--find-renames', base] + names = git(repo, *command, '--name-only', '--diff-filter=AM', '-z', '--', 'Common', 'SerialPrograms') + selected = {} + for raw in filter(None, names.split(b'\0')): + name = os.fsdecode(raw) + if Path(name).suffix not in SUFFIXES: + continue + path = repo / name + if path.is_symlink(): + raise ValueError('Refusing to format a symbolic link: ' + name) + source = path.read_bytes() + diff = git(repo, *command, '--no-color', '--unified=0', '--', ':(literal)' + name) + if path.read_bytes() != source: + raise ValueError('Source changed while measuring the diff: ' + name) + if hunk_lines(diff): + selected[name] = Selection(hunk_lines(diff), source) + return selected + + +def protected_lines(lines): + """Protect preprocessor directives and their backslash continuation lines.""" + protected = set() + continuation = False + for index, line in enumerate(lines): + if continuation or line.lstrip().startswith(b'#'): + protected.add(index + 1) + continuation = line.rstrip(b'\r\n').rstrip(b' \t').endswith(b'\\') + return protected + + +def local_edits(source, formatted, selected): + """Keep same-line whitespace edits in scope; retain indentation and line endings.""" + original = source.splitlines(keepends=True) + output = formatted.splitlines(keepends=True) + result = original.copy() + protected = protected_lines(original) + skipped = Counter() + for tag, start, end, new_start, new_end in difflib.SequenceMatcher( + None, original, output, autojunk=False + ).get_opcodes(): + if tag == 'equal': + continue + if end - start != new_end - new_start: + skipped['reflow'] += len(selected.intersection(range(start + 1, end + 1))) + continue + for index, replacement in zip(range(start, end), output[new_start:new_end]): + if index + 1 not in selected: + continue + old = original[index] + if index + 1 in protected or re.search(rb'\b(class|struct|union)\b', old): + skipped['protected'] += 1 + continue + if b''.join(old.split()) != b''.join(replacement.split()): + raise ValueError('Formatter proposed a non-whitespace rewrite on an edited line.') + body = old.rstrip(b'\r\n') + prefix = body[:len(body) - len(body.lstrip(b' \t'))] + result[index] = prefix + replacement.rstrip(b'\r\n').lstrip(b' \t') + old[len(body):] + return b''.join(result), dict(skipped) + + +def raw_tokens(source, clang): + """Get significant token kinds/spellings without includes, preprocessing or execution.""" + result = subprocess.run( + [clang, '-x', 'c++', '-std=c++23', '-fsyntax-only', '-Xclang', '-dump-raw-tokens', '-'], + input=source, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, + ) + tokens = [] + for record in re.split(rb'\tLoc=<:\d+:\d+>\r?\n', result.stderr): + if not record: + continue + match = re.match(rb"^(\w+) '(.*)'\t", record, re.S) + if match is None: + raise ValueError('Could not parse Clang raw token output; no file was written.') + kind, spelling = match.groups() + if kind == b'comment' or (kind == b'unknown' and spelling.isspace()): + continue + tokens.append((kind, spelling)) + return tokens + + +def verify_tokens(source, candidate, clang): + """Reject whitespace changes that merge identifiers/operators or alter string literals.""" + if raw_tokens(source, clang) != raw_tokens(candidate, clang): + raise ValueError('C++ token kind/spelling changed; no file was written.') + + +def make_plans(repo, selection, uncrustify, clang): + """Verify every candidate before allowing the caller to write any source file.""" + version = subprocess.check_output([uncrustify, '--version'], text=True).strip() + if re.fullmatch(r'Uncrustify-0\.83\.0(?:_f)?', version) is None: + raise ValueError('Expected Uncrustify 0.83.0; found ' + version) + plans = [] + for name, selected in selection.items(): + path = repo / name + if path.is_symlink(): + raise ValueError('Refusing to format a symbolic link: ' + name) + source = selected.source + if path.read_bytes() != source: + raise ValueError('Source changed after measuring the diff: ' + name) + result = subprocess.run( + [uncrustify, '-q', '-c', str(CONFIG), '-l', 'CPP'], input=source, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + if result.returncode or result.stderr: + raise ValueError(name + ': ' + result.stderr.decode(errors='replace')) + candidate, skipped = local_edits(source, result.stdout, selected.lines) + if candidate != source: + verify_tokens(source, candidate, clang) + plans.append(Plan(path, source, candidate, skipped)) + return plans + + +def write_plans(plans): + """Keep the index untouched; refuse plans invalidated by a concurrent source edit.""" + for plan in plans: + if plan.path.is_symlink() or plan.path.read_bytes() != plan.before: + raise ValueError('Source changed after planning: ' + str(plan.path)) + for plan in plans: + if plan.after != plan.before: + if plan.path.is_symlink() or plan.path.read_bytes() != plan.before: + raise ValueError('Source changed before writing: ' + str(plan.path)) + plan.path.write_bytes(plan.after) + + +def main(argv=None): + """Preview by default; --write applies, --check returns 1 for pending local edits.""" + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--repo', type=Path, default=Path.cwd()) + parser.add_argument('--base', default='HEAD', help='Compare the working tree with this revision\'s merge base with HEAD') + parser.add_argument('--uncrustify', default=os.environ.get('UNCRUSTIFY', 'uncrustify')) + parser.add_argument('--clang', default=os.environ.get('CLANG', 'clang')) + mode = parser.add_mutually_exclusive_group() + mode.add_argument('--write', action='store_true') + mode.add_argument('--check', action='store_true') + args = parser.parse_args(argv) + try: + repo = Path(os.fsdecode(git(args.repo, 'rev-parse', '--show-toplevel')).strip()) + base = os.fsdecode(git(repo, 'merge-base', args.base, 'HEAD')).strip() + selection = selected_lines(repo, base) + plans = make_plans(repo, selection, args.uncrustify, args.clang) if selection else [] + for plan in plans: + name = plan.path.relative_to(repo).as_posix() + sys.stdout.writelines(difflib.unified_diff( + plan.before.decode(errors='backslashreplace').splitlines(keepends=True), + plan.after.decode(errors='backslashreplace').splitlines(keepends=True), + fromfile='a/' + name, tofile='b/' + name, + )) + if plan.skipped: + print(name + ': skipped ' + str(plan.skipped), file=sys.stderr) + if args.write: + write_plans(plans) + count = sum(plan.before != plan.after for plan in plans) + print(f'{count} file(s) with eligible edited-line formatting; index untouched.', file=sys.stderr) + return int(args.check and count > 0) + except (OSError, ValueError, subprocess.CalledProcessError) as error: + print('Formatting refused: ' + str(error), file=sys.stderr) + return 2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/devtools/format/test_changed.py b/devtools/format/test_changed.py new file mode 100644 index 0000000000..acf6beefa4 --- /dev/null +++ b/devtools/format/test_changed.py @@ -0,0 +1,149 @@ +"""Incremental formatting tests: python3 -m unittest discover -s devtools/format -v.""" + +from contextlib import redirect_stderr, redirect_stdout +import io +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +import format_changed as changed + + +class LocalEditTests(unittest.TestCase): + def test_only_selected_line_changes_and_indent_is_preserved(self): + source = b' int before=1;\n int value=2;\n int after=3;\n' + formatted = b' int before = 1;\n int value = 2;\n int after = 3;\n' + candidate, _ = changed.local_edits(source, formatted, {2}) + self.assertEqual(b' int before=1;\n int value = 2;\n int after=3;\n', candidate) + + def test_reflows_and_declaration_braces_are_skipped(self): + source = b'class Derived : public Base{\n void f(){ if(x){run();} }\n};\n' + formatted = b'class Derived : public Base {\n void f(){\n if (x){\n run();\n }\n }\n};\n' + candidate, skips = changed.local_edits(source, formatted, {1, 2}) + self.assertEqual(source, candidate) + self.assertTrue(skips) + declaration = b'class Derived : public Base{\n' + candidate, skips = changed.local_edits( + declaration, b'class Derived : public Base {\n', {1} + ) + self.assertEqual(declaration, candidate) + self.assertEqual({'protected': 1}, skips) + + def test_preprocessor_and_continuation_lines_are_not_edited(self): + source = b'#define M(x) \\\n use(x)\nvoid f(){\n run();\n}\n' + formatted = b'#define M(x) \\\n use (x)\nvoid f(){\n run ();\n}\n' + candidate, skips = changed.local_edits(source, formatted, {1, 2, 4}) + self.assertEqual(b'#define M(x) \\\n use(x)\nvoid f(){\n run ();\n}\n', candidate) + self.assertTrue(skips) + + def test_crlf_and_deletion_only_ranges(self): + candidate, _ = changed.local_edits(b'int x=1;\r\n', b'int x = 1;\r\n', {1}) + self.assertEqual(b'int x = 1;\r\n', candidate) + self.assertEqual(set(), changed.hunk_lines(b'@@ -2 +1,0 @@\n-deleted\n')) + self.assertEqual({3, 4}, changed.hunk_lines(b'@@ -2 +3,2 @@\n+x\n+y\n')) + + def test_no_nonwhitespace_rewrite_is_accepted(self): + with self.assertRaisesRegex(ValueError, 'non-whitespace'): + changed.local_edits(b'int x=1;\n', b'int x = 2;\n', {1}) + + +class TokenGuardTests(unittest.TestCase): + def test_identifier_merge_and_literal_changes_are_rejected(self): + for before, after in [ + (b'operator const Type&();\n', b'operator constType&();\n'), + (b'const char* s = "a b";\n', b'const char* s = "ab";\n'), + (b'const char* s = R"x(a b)x";\n', b'const char* s = R"x(ab)x";\n'), + (b'int x = a + +b;\n', b'int x = a++b;\n'), + (b'const char* s = R"x(a\nb)x";\n', b'const char* s = R"x(ab)x";\n'), + ]: + with self.subTest(before=before): + with self.assertRaisesRegex(ValueError, 'token'): + changed.verify_tokens(before, after, 'clang') + + def test_whitespace_in_code_preserves_tokens(self): + changed.verify_tokens(b'if(x){run();}\n', b'if (x){run();}\n', 'clang') + + +class GitIntegrationTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + self.path = self.root / 'Common/example.cpp' + self.path.parent.mkdir() + self.path.write_bytes(b'void f(){\n int old=1;\n int value=2;\n}\n') + self.git('init', '-q') + self.git('add', '.') + self.git('-c', 'user.name=Formatter Test', '-c', 'user.email=test@example.invalid', + '-c', 'core.hooksPath=/dev/null', 'commit', '-qm', 'base') + + def git(self, *args): + return subprocess.check_output(['git', '-C', str(self.root), *args]) + + def cli(self, args): + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + return changed.main(args) + + def test_default_preview_check_and_write_leave_index_untouched(self): + self.path.write_bytes(b'void f(){\n int old=1;\n int value=3;\n}\n') + self.git('add', '.') + staged = self.git('diff', '--cached', '--binary') + source = self.path.read_bytes() + common = ['--repo', str(self.root)] + self.assertEqual(0, self.cli(common)) + self.assertEqual(source, self.path.read_bytes()) + self.assertEqual(1, self.cli([*common, '--check'])) + self.assertEqual(0, self.cli([*common, '--write'])) + self.assertEqual(b'void f(){\n int old=1;\n int value = 3;\n}\n', self.path.read_bytes()) + self.assertEqual(staged, self.git('diff', '--cached', '--binary')) + self.assertEqual(0, self.cli([*common, '--check'])) + + def test_source_change_after_plan_is_refused(self): + plan = changed.Plan(self.path, self.path.read_bytes(), b'new text', {}) + self.path.write_bytes(b'user edit') + with self.assertRaisesRegex(ValueError, 'changed'): + changed.write_plans([plan]) + self.assertEqual(b'user edit', self.path.read_bytes()) + + def test_untracked_and_rename_only_files_are_not_formatted(self): + other = self.root / 'Common/new.cpp' + other.write_bytes(b'int untouched=0;\n') + self.assertEqual({}, changed.selected_lines(self.root, 'HEAD')) + self.git('mv', 'Common/example.cpp', 'Common/renamed.cpp') + self.assertEqual({}, changed.selected_lines(self.root, 'HEAD')) + + def test_filename_with_spaces_and_pathspec_characters(self): + path = self.root / 'Common/new [case].cpp' + path.write_bytes(b'int value=3;\n') + self.git('add', '.') + self.assertEqual(0, self.cli(['--repo', str(self.root), '--write'])) + self.assertEqual(b'int value = 3;\n', path.read_bytes()) + + def test_invalid_second_candidate_prevents_all_writes(self): + self.path.write_bytes(b'int first=1;\n') + other = self.root / 'Common/second.cpp' + other.write_bytes(b'int second=2;\n') + self.git('add', '.') + before = self.path.read_bytes(), other.read_bytes() + with patch.object(changed, 'verify_tokens', side_effect=[None, ValueError('token mismatch')]): + self.assertEqual(2, self.cli(['--repo', str(self.root), '--write'])) + self.assertEqual(before, (self.path.read_bytes(), other.read_bytes())) + + def test_snapshot_change_after_diff_selection_is_refused(self): + self.path.write_bytes(b'int value=3;\n') + selection = changed.selected_lines(self.root, 'HEAD') + self.path.write_bytes(b'user changed the file') + with self.assertRaisesRegex(ValueError, 'changed after measuring'): + changed.make_plans(self.root, selection, 'uncrustify', 'clang') + + def test_explicit_base_includes_committed_changes(self): + base = self.git('rev-parse', 'HEAD').decode().strip() + self.path.write_bytes(b'void f(){\n int old=1;\n int value=3;\n}\n') + self.git('add', '.') + self.git('-c', 'user.name=Formatter Test', '-c', 'user.email=test@example.invalid', + '-c', 'core.hooksPath=/dev/null', 'commit', '-qm', 'edit') + self.assertEqual({}, changed.selected_lines(self.root, 'HEAD')) + self.assertEqual(0, self.cli(['--repo', str(self.root), '--base', base, '--write'])) + self.assertEqual(b'void f(){\n int old=1;\n int value = 3;\n}\n', self.path.read_bytes())