tests.yml

name: Tests

on:
  push:
    branches: [ main, dev, any-llm ]
  pull_request:

permissions:
  contents: read

jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.11", "3.12", "3.13"]

    steps:
      - uses: actions/checkout@v4
        with:
          # testmon diffs against the base to pick affected tests; needs history.
          fetch-depth: 0

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          # dev: test tooling; tracing: aiohttp/grpcio/otlp-proto needed by the
          # builtin OTLP receiver tests.
          pip install -e ".[dev,tracing,inspect,openenv]"

      - name: Cache testmon dependency data
        uses: actions/cache@v4
        with:
          path: .testmondata
          key: testmon-py${{ matrix.python-version }}-${{ github.sha }}
          restore-keys: |
            testmon-py${{ matrix.python-version }}-

      - name: Run tests (parallel, affected-only via testmon) with coverage
        run: |
          # pipefail: without it the step's exit code is tee's, and the job is
          # green whatever pytest reports.
          set -o pipefail
          pytest --testmon -n auto --cov=simpleaudit --cov-report=xml --cov-report=html --cov-report=term-missing --cov-report=term | tee coverage-output.txt
      
      - name: Add coverage summary to job
        if: always()
        run: |
          echo "## Coverage Report - Python ${{ matrix.python-version }}" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          tail -n 20 coverage-output.txt >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
      
      - name: Upload coverage reports
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: coverage-py${{ matrix.python-version }}
          path: |
            coverage.xml
            htmlcov/
          retention-days: 30

  # A full, clean, non-testmon run. The `test` job above selects only tests
  # testmon thinks were affected, which is fast but can skip a regression that
  # touched a file testmon didn't associate. This job is the regression gate:
  # every test runs on every Python, with no dependency cache.
  full-test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.11", "3.12", "3.13"]
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -e ".[dev,tracing,plot,inspect,openenv]"

      - name: Run full test suite (clean, no testmon)
        run: |
          set -o pipefail
          pytest -p no:randomly --cov=simpleaudit --cov-report=xml --cov-report=term-missing | tee full-test-output.txt

      - name: Upload coverage report
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: full-coverage-py${{ matrix.python-version }}
          path: coverage.xml
          retention-days: 30

  # Static analysis. Runs on the single latest interpreter; lint is version-agnostic.
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python 3.13
        uses: actions/setup-python@v5
        with:
          python-version: "3.13"

      - name: Install ruff
        run: |
          python -m pip install --upgrade pip
          pip install "ruff>=0.1.0"

      - name: Check for undefined names and syntax errors
        # E9 (syntax) and F821 (undefined name) are the rules that catch
        # things that actually break. The wider F set (unused imports, etc.) is
        # historical style debt the repo still carries, so gating on it would
        # be red on day one and train people to ignore it.
        run: ruff check --select E9,F821 .

  # Distribution gate: the wheel must build and be importable in a clean
  # interpreter, so a broken pyproject or a missing package module fails here
  # instead of on PyPI.
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Build wheel
        run: |
          python -m pip install --upgrade pip
          pip install build
          python -m build --wheel

      - name: Install built wheel and import it
        run: |
          pip install dist/*.whl
          # Run from outside the checkout (and with -I) so the import resolves
          # to the installed wheel, not ./simpleaudit; load a scenario pack so
          # missing package data fails here too.
          cd "$RUNNER_TEMP"
          python -I -c "
          import simpleaudit, simpleaudit.judges, simpleaudit.scenarios, simpleaudit.targets, simpleaudit.tracing
          from simpleaudit.model_auditor import ModelAuditor
          assert 'site-packages' in simpleaudit.__file__, simpleaudit.__file__
          assert ModelAuditor.get_scenarios('safety')
          print('installed', simpleaudit.__version__, simpleaudit.__file__)
          "