Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# pygrader

## 2.2.0

- Scored checks can now have float max scores

## 2.1.1

- Show pytest stdout as part of the exception
Expand Down
2 changes: 1 addition & 1 deletion CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ This document provides context and guidelines for Claude Sonnet 4.5 when working

### Key Information
- **Language**: Python 3.10+
- **Version**: 2.1.0
- **Version**: 2.2.0
- **License**: GPL-3.0
- **Architecture**: Modular check-based system with CLI and web interfaces
- **Current Branch**: `show-check-results` (default: `main`)
Expand Down
6 changes: 6 additions & 0 deletions docs/source/config.rst
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,12 @@ check's own section is rejected and makes the run fail.
Can be an integer or float value.
The ``structure`` and ``complexity`` checks are pass/fail and do not accept this field.

The ``pylint``, ``coverage`` and ``type-hints`` checks award points in steps: the tool's
result (pylint score 0-10, coverage 0-100%, type-hinted lines 0-100%) is split into
``ceil(max_points) + 1`` equal regions, awarding ``0, step, 2 * step, ..., max_points``
with ``step = max_points / ceil(max_points)``. For example, ``"max_points": 2`` awards
``0``, ``1`` or ``2``, while ``"max_points": 1.5`` awards ``0``, ``0.75`` or ``1.5``.

``is_venv_required`` (required)
Boolean indicating if the check should be run within a virtual environment.
Must be a JSON boolean (``true``/``false``). Strings are not parsed - ``"false"``
Expand Down
34 changes: 32 additions & 2 deletions grader/checks/abstract_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
Each check should inherit from this class.
"""

import itertools
import logging
import math
from abc import ABC, abstractmethod

from grader.exceptions import CheckError
Expand Down Expand Up @@ -109,7 +111,7 @@ class ScoredCheck(AbstractCheck[float]):
def __init__(
self,
name: str,
max_points: int,
max_points: float,
project_root: str,
is_venv_requred: bool = False,
env_vars: dict[str, str] | None = None,
Expand All @@ -129,10 +131,38 @@ def __init__(
self._max_points = max_points

@property
def max_points(self) -> int:
def max_points(self) -> float:
"""Return the maximum amount of points that can be achieved by the check."""
return self._max_points

def _translate_score(self, value: float, value_max: float) -> float:
"""
Split ``[0, value_max]`` into equal regions and award points based on the region ``value`` falls in.

There are ``ceil(max_points) + 1`` regions, awarding ``0, step, 2 * step, ..., max_points``
where ``step = max_points / ceil(max_points)``. For an integer ``max_points`` the step is 1.

:param value: The raw value produced by the tool (e.g. pylint score, coverage percentage).
:param value_max: The maximum raw value the tool can produce.
:raises CheckError: If max points is set to -1.
:return: The translated score.
"""
if self._max_points == -1:
raise CheckError(f"Max points for {self.name} check is set to -1")

region_count = math.ceil(self._max_points)
points_per_region = self._max_points / region_count if region_count else 0

step = value_max / (region_count + 1)
steps = [i * step for i in range(region_count + 2)]

for region, (start, end) in enumerate(itertools.pairwise(steps)):
if round(start, 2) <= round(value, 2) < round(end, 2):
# Keep integer scores for integer max points
return region if points_per_region == 1 else round(region * points_per_region, 2)

return self._max_points


class NonScoredCheck(AbstractCheck[bool]):
"""Non-scored checks do not have a maximum amount of points."""
Expand Down
28 changes: 2 additions & 26 deletions grader/checks/coverage_check.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
"""Module containing the unit test code coverage check."""

import itertools
import logging

from grader.checks.abstract_check import ScoredCheck
Expand All @@ -26,7 +25,7 @@ def __init__(
self,
name: str,
project_root: str,
max_points: int,
max_points: float,
is_venv_required: bool,
env_vars: dict[str, str] | None = None,
assets: list[str] | None = None,
Expand Down Expand Up @@ -61,34 +60,11 @@ def run(self) -> ScoredCheckResult:
if coverage_report_result is None:
raise CheckError("Coverage report generation failed")

score = self.__translate_score(coverage_report_result)
score = self._translate_score(coverage_report_result, 100)
return ScoredCheckResult(
self.name, score, f"Tests cover {coverage_report_result:.2f}% of the code", "", self.max_points
)

def __translate_score(self, coverage_score: float) -> float:
"""
Split the coverage score into regions and assign a score based on the region.

The amount of regions depends on the max points for the criteria.

:param coverage_score: The score from pylint to be translated
:return: The translated score
"""
if self._max_points == -1:
raise CheckError("Max points for coverage check is set to -1")

step = 100 / (self._max_points + 1)
steps = [i * step for i in range(self._max_points + 2)]

regions = list(itertools.pairwise(steps))

for score, (start, end) in enumerate(regions):
if round(start, 2) <= round(coverage_score, 2) < round(end, 2):
return score

return self._max_points

def __coverage_run(self) -> None:
"""Run the coverage tool on the project."""
command = [self.__coverage_full_path] + COVERAGE_RUN_ARGS + COVERAGE_RUN_PYTEST_ARGS
Expand Down
28 changes: 2 additions & 26 deletions grader/checks/pylint_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
It uses the pylint python library directly to run the check.
"""

import itertools
import logging
import re
from io import StringIO
Expand All @@ -29,7 +28,7 @@ def __init__(
self,
name: str,
project_root: str,
max_points: int,
max_points: float,
is_venv_required: bool,
pylintrc_path: str | None = None,
env_vars: dict[str, str] | None = None,
Expand Down Expand Up @@ -88,35 +87,12 @@ def run(self) -> ScoredCheckResult:
pylint_score = self.__get_pylint_score(results.stdout)

logger.debug("Pylint score: %s", pylint_score)
score = self.__translate_score(pylint_score)
score = self._translate_score(pylint_score, self.__pylint_max_score)

short_output = self.__summarize_output(results.stdout)

return ScoredCheckResult(self.name, score, short_output, "", self.max_points)

def __translate_score(self, pylint_score: float) -> float:
"""
Split the pylint score into regions and assign a score based on the region.

The amount of regions depends on the max points for the criteria.

:param pylint_score: The score from pylint to be translated
:return: The translated score
"""
if self._max_points == -1:
raise CheckError("Max points for pylint check is set to -1")

step = self.__pylint_max_score / (self._max_points + 1)
steps = [i * step for i in range(self._max_points + 2)]

regions = list(itertools.pairwise(steps))

for score, (start, end) in enumerate(regions):
if round(start, 2) <= round(pylint_score, 2) < round(end, 2):
return score

return self._max_points

def __get_pylint_score(self, pylint_output: str) -> float:
"""
Get the score from the pylint output.
Expand Down
2 changes: 1 addition & 1 deletion grader/checks/requirements_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def __init__(
self,
name: str,
project_root: str,
max_points: int,
max_points: float,
is_venv_required: bool,
is_checking_install: bool = False,
env_vars: dict[str, str] | None = None,
Expand Down
2 changes: 1 addition & 1 deletion grader/checks/run_tests_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def __init__(
self,
name: str,
project_root: str,
max_points: int,
max_points: float,
is_venv_required: bool,
tests_path: list[str],
default_test_score: float = 0.0,
Expand Down
2 changes: 1 addition & 1 deletion grader/checks/spec_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def __init__(
self,
name: str,
project_root: str,
max_points: int,
max_points: float,
is_venv_required: bool = False,
env_vars: dict[str, str] | None = None,
assets: list[str] | None = None,
Expand Down
28 changes: 2 additions & 26 deletions grader/checks/type_hints_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
It calls mypy as a subprocess to generate a report and then read from the report.
"""

import itertools
import logging

from grader.checks.abstract_check import ScoredCheck
Expand All @@ -23,7 +22,7 @@ def __init__(
self,
name: str,
project_root: str,
max_points: int,
max_points: float,
is_venv_required: bool,
env_vars: dict[str, str] | None = None,
assets: list[str] | None = None,
Expand Down Expand Up @@ -95,7 +94,7 @@ def run(self) -> ScoredCheckResult:

# Calculate score
covered_lines_percentage = int(lines_with_type_annotations) / int(lines_total)
score = self.__translate_score(covered_lines_percentage)
score = self._translate_score(covered_lines_percentage, self.__mypy_max_score)

return ScoredCheckResult(
self.name,
Expand All @@ -104,26 +103,3 @@ def run(self) -> ScoredCheckResult:
"",
self.max_points,
)

def __translate_score(self, mypy_score: float) -> float:
"""
Split the mypy score into regions and assign a score based on the region.

The amount of regions depends on the max points for the criteria.

:param pylint_score: The score from pylint to be translated
:return: The translated score
"""
if self._max_points == -1:
raise CheckError("Max points for type hints check is set to -1")

step = self.__mypy_max_score / (self._max_points + 1)
steps = [i * step for i in range(self._max_points + 2)]

regions = list(itertools.pairwise(steps))

for score, (start, end) in enumerate(regions):
if round(start, 2) <= round(mypy_score, 2) < round(end, 2):
return score

return self._max_points
2 changes: 1 addition & 1 deletion grader/models/check_result.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ class CheckResult[T]:
class ScoredCheckResult(CheckResult[float]):
"""Class representing the result of a scored check."""

max_score: int
max_score: float


@dataclass
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "pygrader"
version = "2.1.1"
version = "2.2.0"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.12"
Expand Down
Loading
Loading