From b6d99e34f952603b3ba023dd9eaa4006ca7567a6 Mon Sep 17 00:00:00 2001 From: Lyuboslav Karev Date: Mon, 16 Nov 2026 22:30:20 +0200 Subject: [PATCH 1/3] Add support for float max_score in pylint, mypy and coverage checks --- CLAUDE.md | 2 +- docs/source/config.rst | 6 +++ grader/checks/abstract_check.py | 34 +++++++++++++++- grader/checks/coverage_check.py | 28 +------------ grader/checks/pylint_check.py | 28 +------------ grader/checks/requirements_check.py | 2 +- grader/checks/run_tests_check.py | 2 +- grader/checks/spec_check.py | 2 +- grader/checks/type_hints_check.py | 28 +------------ grader/models/check_result.py | 2 +- pyproject.toml | 2 +- tests/unit/test_abstract_check.py | 62 ++++++++++++++++++++++++++++- tests/unit/test_coverage_check.py | 16 ++++++++ uv.lock | 4 +- 14 files changed, 128 insertions(+), 90 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d34c4a7..0fcda5e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ This document provides context and guidelines for Claude Sonnet 4.5 when working ### Key Information - **Language**: Python 3.10+ -- **Version**: 2.1.0 +- **Version**: 2.2.0 - **License**: GPL-3.0 - **Architecture**: Modular check-based system with CLI and web interfaces - **Current Branch**: `show-check-results` (default: `main`) diff --git a/docs/source/config.rst b/docs/source/config.rst index 4347a10..208a27a 100644 --- a/docs/source/config.rst +++ b/docs/source/config.rst @@ -107,6 +107,12 @@ check's own section is rejected and makes the run fail. Can be an integer or float value. The ``structure`` and ``complexity`` checks are pass/fail and do not accept this field. + The ``pylint``, ``coverage`` and ``type-hints`` checks award points in steps: the tool's + result (pylint score 0-10, coverage 0-100%, type-hinted lines 0-100%) is split into + ``ceil(max_points) + 1`` equal regions, awarding ``0, step, 2 * step, ..., max_points`` + with ``step = max_points / ceil(max_points)``. For example, ``"max_points": 2`` awards + ``0``, ``1`` or ``2``, while ``"max_points": 1.5`` awards ``0``, ``0.75`` or ``1.5``. + ``is_venv_required`` (required) Boolean indicating if the check should be run within a virtual environment. Must be a JSON boolean (``true``/``false``). Strings are not parsed - ``"false"`` diff --git a/grader/checks/abstract_check.py b/grader/checks/abstract_check.py index 4013c8c..400e675 100644 --- a/grader/checks/abstract_check.py +++ b/grader/checks/abstract_check.py @@ -4,7 +4,9 @@ Each check should inherit from this class. """ +import itertools import logging +import math from abc import ABC, abstractmethod from grader.exceptions import CheckError @@ -109,7 +111,7 @@ class ScoredCheck(AbstractCheck[float]): def __init__( self, name: str, - max_points: int, + max_points: float, project_root: str, is_venv_requred: bool = False, env_vars: dict[str, str] | None = None, @@ -129,10 +131,38 @@ def __init__( self._max_points = max_points @property - def max_points(self) -> int: + def max_points(self) -> float: """Return the maximum amount of points that can be achieved by the check.""" return self._max_points + def _translate_score(self, value: float, value_max: float) -> float: + """ + Split ``[0, value_max]`` into equal regions and award points based on the region ``value`` falls in. + + There are ``ceil(max_points) + 1`` regions, awarding ``0, step, 2 * step, ..., max_points`` + where ``step = max_points / ceil(max_points)``. For an integer ``max_points`` the step is 1. + + :param value: The raw value produced by the tool (e.g. pylint score, coverage percentage). + :param value_max: The maximum raw value the tool can produce. + :raises CheckError: If max points is set to -1. + :return: The translated score. + """ + if self._max_points == -1: + raise CheckError(f"Max points for {self.name} check is set to -1") + + region_count = math.ceil(self._max_points) + points_per_region = self._max_points / region_count if region_count else 0 + + step = value_max / (region_count + 1) + steps = [i * step for i in range(region_count + 2)] + + for region, (start, end) in enumerate(itertools.pairwise(steps)): + if round(start, 2) <= round(value, 2) < round(end, 2): + # Keep integer scores for integer max points + return region if points_per_region == 1 else round(region * points_per_region, 2) + + return self._max_points + class NonScoredCheck(AbstractCheck[bool]): """Non-scored checks do not have a maximum amount of points.""" diff --git a/grader/checks/coverage_check.py b/grader/checks/coverage_check.py index 2d37dae..d437ee6 100644 --- a/grader/checks/coverage_check.py +++ b/grader/checks/coverage_check.py @@ -1,6 +1,5 @@ """Module containing the unit test code coverage check.""" -import itertools import logging from grader.checks.abstract_check import ScoredCheck @@ -26,7 +25,7 @@ def __init__( self, name: str, project_root: str, - max_points: int, + max_points: float, is_venv_required: bool, env_vars: dict[str, str] | None = None, assets: list[str] | None = None, @@ -61,34 +60,11 @@ def run(self) -> ScoredCheckResult: if coverage_report_result is None: raise CheckError("Coverage report generation failed") - score = self.__translate_score(coverage_report_result) + score = self._translate_score(coverage_report_result, 100) return ScoredCheckResult( self.name, score, f"Tests cover {coverage_report_result:.2f}% of the code", "", self.max_points ) - def __translate_score(self, coverage_score: float) -> float: - """ - Split the coverage score into regions and assign a score based on the region. - - The amount of regions depends on the max points for the criteria. - - :param coverage_score: The score from pylint to be translated - :return: The translated score - """ - if self._max_points == -1: - raise CheckError("Max points for coverage check is set to -1") - - step = 100 / (self._max_points + 1) - steps = [i * step for i in range(self._max_points + 2)] - - regions = list(itertools.pairwise(steps)) - - for score, (start, end) in enumerate(regions): - if round(start, 2) <= round(coverage_score, 2) < round(end, 2): - return score - - return self._max_points - def __coverage_run(self) -> None: """Run the coverage tool on the project.""" command = [self.__coverage_full_path] + COVERAGE_RUN_ARGS + COVERAGE_RUN_PYTEST_ARGS diff --git a/grader/checks/pylint_check.py b/grader/checks/pylint_check.py index ba1a733..98e6b40 100644 --- a/grader/checks/pylint_check.py +++ b/grader/checks/pylint_check.py @@ -3,7 +3,6 @@ It uses the pylint python library directly to run the check. """ -import itertools import logging import re from io import StringIO @@ -29,7 +28,7 @@ def __init__( self, name: str, project_root: str, - max_points: int, + max_points: float, is_venv_required: bool, pylintrc_path: str | None = None, env_vars: dict[str, str] | None = None, @@ -88,35 +87,12 @@ def run(self) -> ScoredCheckResult: pylint_score = self.__get_pylint_score(results.stdout) logger.debug("Pylint score: %s", pylint_score) - score = self.__translate_score(pylint_score) + score = self._translate_score(pylint_score, self.__pylint_max_score) short_output = self.__summarize_output(results.stdout) return ScoredCheckResult(self.name, score, short_output, "", self.max_points) - def __translate_score(self, pylint_score: float) -> float: - """ - Split the pylint score into regions and assign a score based on the region. - - The amount of regions depends on the max points for the criteria. - - :param pylint_score: The score from pylint to be translated - :return: The translated score - """ - if self._max_points == -1: - raise CheckError("Max points for pylint check is set to -1") - - step = self.__pylint_max_score / (self._max_points + 1) - steps = [i * step for i in range(self._max_points + 2)] - - regions = list(itertools.pairwise(steps)) - - for score, (start, end) in enumerate(regions): - if round(start, 2) <= round(pylint_score, 2) < round(end, 2): - return score - - return self._max_points - def __get_pylint_score(self, pylint_output: str) -> float: """ Get the score from the pylint output. diff --git a/grader/checks/requirements_check.py b/grader/checks/requirements_check.py index e0a3875..2db438c 100644 --- a/grader/checks/requirements_check.py +++ b/grader/checks/requirements_check.py @@ -23,7 +23,7 @@ def __init__( self, name: str, project_root: str, - max_points: int, + max_points: float, is_venv_required: bool, is_checking_install: bool = False, env_vars: dict[str, str] | None = None, diff --git a/grader/checks/run_tests_check.py b/grader/checks/run_tests_check.py index 99f0105..87f220f 100644 --- a/grader/checks/run_tests_check.py +++ b/grader/checks/run_tests_check.py @@ -43,7 +43,7 @@ def __init__( self, name: str, project_root: str, - max_points: int, + max_points: float, is_venv_required: bool, tests_path: list[str], default_test_score: float = 0.0, diff --git a/grader/checks/spec_check.py b/grader/checks/spec_check.py index 376ebea..37b3da1 100644 --- a/grader/checks/spec_check.py +++ b/grader/checks/spec_check.py @@ -52,7 +52,7 @@ def __init__( self, name: str, project_root: str, - max_points: int, + max_points: float, is_venv_required: bool = False, env_vars: dict[str, str] | None = None, assets: list[str] | None = None, diff --git a/grader/checks/type_hints_check.py b/grader/checks/type_hints_check.py index 09e3b4f..cd14787 100644 --- a/grader/checks/type_hints_check.py +++ b/grader/checks/type_hints_check.py @@ -4,7 +4,6 @@ It calls mypy as a subprocess to generate a report and then read from the report. """ -import itertools import logging from grader.checks.abstract_check import ScoredCheck @@ -23,7 +22,7 @@ def __init__( self, name: str, project_root: str, - max_points: int, + max_points: float, is_venv_required: bool, env_vars: dict[str, str] | None = None, assets: list[str] | None = None, @@ -95,7 +94,7 @@ def run(self) -> ScoredCheckResult: # Calculate score covered_lines_percentage = int(lines_with_type_annotations) / int(lines_total) - score = self.__translate_score(covered_lines_percentage) + score = self._translate_score(covered_lines_percentage, self.__mypy_max_score) return ScoredCheckResult( self.name, @@ -104,26 +103,3 @@ def run(self) -> ScoredCheckResult: "", self.max_points, ) - - def __translate_score(self, mypy_score: float) -> float: - """ - Split the mypy score into regions and assign a score based on the region. - - The amount of regions depends on the max points for the criteria. - - :param pylint_score: The score from pylint to be translated - :return: The translated score - """ - if self._max_points == -1: - raise CheckError("Max points for type hints check is set to -1") - - step = self.__mypy_max_score / (self._max_points + 1) - steps = [i * step for i in range(self._max_points + 2)] - - regions = list(itertools.pairwise(steps)) - - for score, (start, end) in enumerate(regions): - if round(start, 2) <= round(mypy_score, 2) < round(end, 2): - return score - - return self._max_points diff --git a/grader/models/check_result.py b/grader/models/check_result.py index 0d7184d..167e1d5 100644 --- a/grader/models/check_result.py +++ b/grader/models/check_result.py @@ -17,7 +17,7 @@ class CheckResult[T]: class ScoredCheckResult(CheckResult[float]): """Class representing the result of a scored check.""" - max_score: int + max_score: float @dataclass diff --git a/pyproject.toml b/pyproject.toml index 411b15d..28da98f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pygrader" -version = "2.1.1" +version = "2.2.0" description = "Add your description here" readme = "README.md" requires-python = ">=3.12" diff --git a/tests/unit/test_abstract_check.py b/tests/unit/test_abstract_check.py index 62a0e3e..b1b394f 100644 --- a/tests/unit/test_abstract_check.py +++ b/tests/unit/test_abstract_check.py @@ -4,9 +4,9 @@ import pytest -from grader.checks.abstract_check import AbstractCheck +from grader.checks.abstract_check import AbstractCheck, ScoredCheck from grader.exceptions import CheckError -from grader.models.check_result import CheckResult +from grader.models.check_result import CheckResult, ScoredCheckResult class DummyCheck(AbstractCheck): @@ -50,3 +50,61 @@ def test_03_assets_are_wrapped_in_resource(self, mocked_resource: MagicMock) -> # Assert mocked_resource.assert_has_calls([call(source) for source in sources]) assert check.assets == [mocked_resource.return_value, mocked_resource.return_value] + + +class DummyScoredCheck(ScoredCheck): + """Dummy scored check class for testing the score translation.""" + + def run(self) -> ScoredCheckResult: + """Run the dummy scored check.""" + return ScoredCheckResult(self.name, 0, "", "", self.max_points) + + +class TestScoredCheckTranslateScore: + """Test cases for ScoredCheck._translate_score.""" + + @pytest.mark.parametrize( + ("max_points", "value", "expected"), + [ + # Integer max points: 3 regions of ~33.33, awarding 0, 1, 2 + (2, 0, 0), + (2, 33.33, 1), + (2, 50, 1), + (2, 66.67, 2), + (2, 100, 2), + # 1.5 -> 3 regions of ~33.33, awarding 0, 0.75, 1.5 + (1.5, 0, 0), + (1.5, 33.32, 0), + (1.5, 33.33, 0.75), + (1.5, 66.67, 1.5), + (1.5, 100, 1.5), + # 0.5 -> 2 regions of 50, awarding 0, 0.5 + (0.5, 49.99, 0), + (0.5, 50, 0.5), + (0.5, 100, 0.5), + # 2.5 -> 4 regions of 25, awarding 0, 0.83, 1.67, 2.5 + (2.5, 24.99, 0), + (2.5, 25, 0.83), + (2.5, 50, 1.67), + (2.5, 75, 2.5), + (2.5, 100, 2.5), + ], + ) + def test_01_translate_score(self, max_points: float, value: float, expected: float) -> None: + """Test that a raw value is translated into the expected score for integer and float max points.""" + check = DummyScoredCheck("dummy", max_points, "dummy") + + assert check._translate_score(value, 100) == expected # pylint: disable=protected-access + + def test_02_translate_score_integer_max_returns_int(self) -> None: + """Test that integer max points keep producing integer scores.""" + check = DummyScoredCheck("dummy", 2, "dummy") + + assert isinstance(check._translate_score(50, 100), int) # pylint: disable=protected-access + + def test_03_translate_score_negative_max(self) -> None: + """Test that max points of -1 raise a CheckError.""" + check = DummyScoredCheck("dummy", -1, "dummy") + + with pytest.raises(CheckError): + check._translate_score(50, 100) # pylint: disable=protected-access diff --git a/tests/unit/test_coverage_check.py b/tests/unit/test_coverage_check.py index e127d18..ecf678f 100644 --- a/tests/unit/test_coverage_check.py +++ b/tests/unit/test_coverage_check.py @@ -194,3 +194,19 @@ def mocked_run_side_effect(*args: list, **_: dict) -> CompletedProcess | None: # Assert assert ScoredCheckResult("Coverage", 2, "Tests cover 100.00% of the code", "", 2) == result + + @patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_run") + @patch("grader.checks.coverage_check.CoverageCheck._CoverageCheck__coverage_report") + def test_float_max_points(self, mocked_report: MagicMock, mocked_run: MagicMock) -> None: + """Test that a float max points value is scored in scaled steps.""" + # Arrange + check = CoverageCheck("Coverage", "sample_dir", 1.5, is_venv_required=False) + mocked_run.return_value = True + mocked_report.return_value = 50 + expected_score = ScoredCheckResult("Coverage", 0.75, "Tests cover 50.00% of the code", "", 1.5) + + # Act + actual_score = check.run() + + # Assert + assert expected_score == actual_score diff --git a/uv.lock b/uv.lock index 6948a32..99f1a85 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.15'", @@ -1027,7 +1027,7 @@ wheels = [ [[package]] name = "pygrader" -version = "2.1.1" +version = "2.2.0" source = { editable = "." } dependencies = [ { name = "complexipy" }, From a62310241b2fc8038e09068a30ffa2508605190f Mon Sep 17 00:00:00 2001 From: Lyuboslav Karev Date: Mon, 16 Nov 2026 22:35:18 +0200 Subject: [PATCH 2/3] Add SKILL.md for creating pygrader configs --- skills/pygrader-config/SKILL.md | 151 ++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 skills/pygrader-config/SKILL.md diff --git a/skills/pygrader-config/SKILL.md b/skills/pygrader-config/SKILL.md new file mode 100644 index 0000000..61daf7d --- /dev/null +++ b/skills/pygrader-config/SKILL.md @@ -0,0 +1,151 @@ +--- +name: pygrader-config +description: Create or edit a pygrader grading configuration (the JSON file passed via --config). Use when the user wants a new grading config, to add/change checks or point values, to set up a structure file, test scoring, venv options, environment variables or a spec-graded check. +metadata: + version: 1.0.0 + pygrader-version: ">=2.2.0" +--- + +# Creating a pygrader configuration + +A grading config is a JSON file passed to pygrader with `--config`. It lists the checks to +run, how many points each is worth, and their options. The full reference is +`docs/source/config.rst`. When this skill and that file disagree, the file wins: update this +skill and bump its version. + +## Workflow + +1. Ask what should be graded (lint, type hints, coverage, the course's own tests, + project structure, requirements, LLM spec grading) and the points for each, unless already given. +2. Write the config to `config/.json`. Put companion files (structure JSON, pylintrc, + spec.md) next to it and reference them with `${{config_dir}}`. +3. Validate it (see **Validate** below) before handing it over. + +## Root object + +```json +{ + "environment": { "variables": { "KEY": "value" } }, + "venv": { "is_keeping_existing_venv": false, "name": ".venv-pygrader" }, + "checks": [ ... ] +} +``` + +- `checks` (required): array of check objects; they run in this order. +- `environment.variables` (optional): env vars for every check. A check's own + `environment.variables` overrides them. +- `venv` (optional): only the keys `is_keeping_existing_venv` (bool, default `false`) and + `name` (default `.venv-pygrader`). Any other key fails the run. A venv is only created when some + check has `"is_venv_required": true`. +- Other unknown root keys are **silently ignored**, so a typo goes unnoticed. Check the spelling. + +## Common check keys + +| Key | Required | Notes | +|---|---|---| +| `name` | yes | Check type: `requirements`, `pylint`, `type-hints`, `coverage`, `tests`, `structure`, `complexity`, `spec` | +| `is_venv_required` | yes | JSON boolean. `"false"` (a string) counts as `true` | +| `max_points` | scored checks | int or float. **Not allowed** on `structure` and `complexity` (pass/fail) | +| `environment` | no | `{ "variables": {...} }`, overrides the global variables | +| `assets` | no | Array of local paths / URLs / `cove://` URIs | + +Any key not listed here or in the check's own section **fails the run**: it's passed +to the check's constructor as a keyword argument. + +## Per-check options + +- **requirements**: `is_checking_install` (bool, default `false`) also checks that + `requirements.txt` installs. Without it, the check only looks for the file. Awards all or nothing. +- **pylint**: `pylintrc_path` (local path only; default is the bundled `config/2024.pylintrc`). + Needs `"is_venv_required": true` so pylint can resolve the project's imports. +- **type-hints**: no options. Scores the share of lines with type annotations (via mypy). +- **coverage**: no options. Runs the project's tests under coverage. Needs `"is_venv_required": true`. +- **tests**: runs course-provided test files against the project. Needs `"is_venv_required": true`. + - `tests_path` (required): array of local paths, URLs or `cove://` Python items. + - `default_test_score` (default `0`): points for each test that has no mapping. + - `test_score_mapping`: `{ "TestClass": 0.5, "test_func": 2 }`. A class value applies to + **each** test in that class; a test-name entry takes precedence over its class entry. + - The weights of **all** tests (passed + failed) must add up to ≤ `max_points`, otherwise the check + errors. Make them add up to exactly `max_points`. +- **structure** (pass/fail): `structure_file` (required; path, URL or `cove://` JSON item), + `is_fatal` (bool, stored but not acted on yet). +- **complexity** (pass/fail): `max_complexity` (default `15`), `is_fatal`. +- **spec** (LLM grading): the first `assets` entry must be the markdown spec. Optional + `model`, `base_url` (OpenAI-compatible, default OpenRouter), `api_key_env` (default + `OPENROUTER_API_KEY`). The API key must be set in the environment / `.env`. + +## How points are awarded + +`pylint`, `coverage` and `type-hints` award points in **steps**, not proportionally. The tool's +result (pylint 0-10, coverage 0-100%, type hints 0-100%) is split into `ceil(max_points) + 1` +equal ranges, which award `0, step, ..., max_points`, where `step = max_points / ceil(max_points)`. + +- `max_points: 3` → 4 ranges → `0 / 1 / 2 / 3` (coverage ≥ 75% gets full marks) +- `max_points: 1.5` → 3 ranges → `0 / 0.75 / 1.5` + +So a large `max_points` makes the scoring finer-grained. Explain this to the user when they pick values. + +## Structure file + +A separate JSON file referenced by `structure_file`. Each top-level key is one category: + +```json +{ + "source": { "name": "Source files", "required": true, "patterns": ["src/**/*.py"] }, + "tests": { "name": "Test files", "required": false, "patterns": ["tests/**/*.py"] }, + "main": { "name": "Main file", "required": true, "patterns": ["main.py"] } +} +``` + +## Remote resources and templates + +- `${{config_dir}}` expands to the config's directory (for remote/Cove configs, the temp dir it was + downloaded to). **Any other `${{...}}` placeholder is an error.** +- The config itself, `tests_path` and `structure_file` accept `http(s)`/`ftp` URLs and `cove://` + URIs. `pylintrc_path` accepts only a local path. Cove needs `COVE_API_KEY`. + +## Example + +```json +{ + "venv": { "is_keeping_existing_venv": true }, + "checks": [ + { "name": "structure", "is_venv_required": false, + "structure_file": "${{config_dir}}/project_structure.json" }, + { "name": "requirements", "max_points": 1, "is_venv_required": false, "is_checking_install": true }, + { "name": "pylint", "max_points": 3, "is_venv_required": true, + "pylintrc_path": "${{config_dir}}/2024.pylintrc" }, + { "name": "type-hints", "max_points": 1.5, "is_venv_required": true }, + { "name": "coverage", "max_points": 5, "is_venv_required": true }, + { "name": "tests", "max_points": 6, "is_venv_required": true, + "tests_path": ["${{config_dir}}/tests/test_solution.py"], + "default_test_score": 1, + "test_score_mapping": { "TestEdgeCases": 0.5 } }, + { "name": "complexity", "is_venv_required": false, "max_complexity": 15 } + ] +} +``` + +More examples live in `config/` (`full.json`, `projects_2026_summer.json`, `spec_example.json`, +`templates_example.json`). + +## Validate + +Build the checks without grading anything. This catches unknown check names, missing required keys, +bad options and bad `${{...}}` templates: + +```bash +uv run python -c "from grader.utils.config import load_config; from grader.checks.checks_factory import create_checks; c = load_config('config/.json'); print(create_checks(c, '.'))" +``` + +Then do a real run on a sample project (`just setup_sample_project` clones one): + +```bash +uv run python pygrader.py --config config/.json -v +``` + +Use `--checks pylint,coverage` to run a subset while iterating. + +## Changelog + +- **1.0.0**: Initial version, matching pygrader 2.2.0 (float `max_points` with scaled steps). From 1f97c1e1bd20e69fb685d83076870a5c6b10d225 Mon Sep 17 00:00:00 2001 From: Lyuboslav Karev Date: Thu, 19 Nov 2026 23:51:13 +0200 Subject: [PATCH 3/3] Update CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3efb227..f3b161c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # pygrader +## 2.2.0 + +- Scored checks can now have float max scores + ## 2.1.1 - Show pytest stdout as part of the exception