Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion .github/actions/run-service-tests/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ inputs:
uv-version:
description: 'uv version to use'
required: true
pytest-args:
description: 'Extra arguments passed to pytest through `make test-all`'
required: false
default: ''

runs:
using: 'composite'
Expand Down Expand Up @@ -61,5 +65,7 @@ runs:

- name: Run full test suite
shell: bash
env:
PYTEST_ARGS: ${{ inputs.pytest-args }}
run: |
make test-all
make test-all ARGS="$PYTEST_ARGS"
178 changes: 176 additions & 2 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
with:
python-version: ${{ env.PYTHON_VERSION }}
uv-version: ${{ env.UV_VERSION }}
pytest-args: --cov --cov-report=
env:
HF_HOME: ${{ github.workspace }}/hf_cache
HF_TOKEN: ${{ secrets.HF_TOKEN }}
Expand Down Expand Up @@ -61,6 +62,17 @@
LANGCACHE_NO_ATTRIBUTES_API_KEY: ${{ secrets.LANGCACHE_NO_ATTRIBUTES_API_KEY }}
LANGCACHE_NO_ATTRIBUTES_CACHE_ID: ${{ secrets.LANGCACHE_NO_ATTRIBUTES_CACHE_ID }}
LANGCACHE_NO_ATTRIBUTES_URL: ${{ secrets.LANGCACHE_NO_ATTRIBUTES_URL }}
COVERAGE_FILE: .coverage.service

- name: Upload coverage data
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v7
with:
name: coverage-data-service
path: .coverage.service
include-hidden-files: true # the data file is a dotfile
overwrite: true # a re-run attempt replaces its earlier upload
retention-days: 7

test:
name: Python ${{ matrix.python-version }} - redis-py ${{ matrix.redis-py-version }} [${{ matrix.redis-image }}]
Expand Down Expand Up @@ -140,9 +152,159 @@

- name: Run tests
run: |
make test
# One data file per leg; the coverage job combines them. Artifact
# names cannot contain ':', so `redis:8.4` becomes `redis-8.4`.
leg="core-py${{ matrix.python-version }}-redis-py-${REDIS_PY_VERSION}-${REDIS_IMAGE//:/-}"
echo "COVERAGE_LEG=$leg" >> "$GITHUB_ENV"
COVERAGE_FILE=".coverage.$leg" make test ARGS="--cov --cov-report="

- name: Upload coverage data
if: ${{ !cancelled() && env.COVERAGE_LEG != '' }}
uses: actions/upload-artifact@v7
with:
name: coverage-data-${{ env.COVERAGE_LEG }}
path: .coverage.${{ env.COVERAGE_LEG }}
include-hidden-files: true # the data file is a dotfile
overwrite: true # a re-run attempt replaces its earlier upload
retention-days: 7
Comment thread
vishal-bala marked this conversation as resolved.
Resolved

coverage:
name: Coverage
runs-on: ubuntu-latest
# Runs whenever the test jobs finish, pass or fail: coverage from a red run
# is still a measurement. A skipped job (service-tests on fork PRs,
# cluster-tests on every PR and push) just leaves its segment out, and
# Codecov carries that segment forward.
needs: [test, service-tests, cluster-tests]
if: ${{ !cancelled() }}
steps:
- name: Check out repository
uses: actions/checkout@v6

- name: Install Python
uses: actions/setup-python@v6
with:
python-version: ${{ env.PYTHON_VERSION }}

- name: Install uv
uses: astral-sh/setup-uv@v10.0.0
with:
version: ${{ env.UV_VERSION }}
enable-cache: true
python-version: ${{ env.PYTHON_VERSION }} # sets UV_PYTHON
cache-dependency-glob: |
pyproject.toml
uv.lock

- name: Install coverage
# The dev group pins coverage through pytest-cov, so this reads the
# data with the same coverage version that wrote it.
run: |
uv sync --frozen --only-group dev --no-install-project

- name: Download coverage data
uses: actions/download-artifact@v8
with:
pattern: coverage-data-* # not coverage-html from an earlier attempt
path: coverage-data
merge-multiple: true

- name: Combine coverage
id: combine
run: |
set -euo pipefail
shopt -s nullglob
cov() { uv run --no-sync coverage "$@"; }

{
echo "## Coverage"
echo
echo "| Segment | Data files | Total |"
echo "|---|---:|---:|"
} >> "$GITHUB_STEP_SUMMARY"

# Segment data lives in its own directory: `patch = ["subprocess"]`
# turns on parallel mode, so reporting on ./.coverage would otherwise
# absorb any ./.coverage.* file beside it.
mkdir -p coverage-segments
segments=()
# A new segment also needs its job in `needs` above, an upload step
# below, and a carryforward entry in codecov.yml if PRs can skip it.
for seg in core service cluster; do
files=(coverage-data/.coverage."$seg"*)
if [ ${#files[@]} -eq 0 ]; then
echo "| $seg | 0 | not run |" >> "$GITHUB_STEP_SUMMARY"
continue
fi
data="coverage-segments/$seg"
cov combine --keep --data-file="$data" "${files[@]}"
cov xml --data-file="$data" -o "coverage-$seg.xml"
echo "| $seg | ${#files[@]} | $(cov report --data-file="$data" --format=total)% |" >> "$GITHUB_STEP_SUMMARY"
echo "$seg=true" >> "$GITHUB_OUTPUT"
segments+=("$data")
done

if [ ${#segments[@]} -eq 0 ]; then
echo "::warning title=No coverage data::No test job uploaded coverage data in this run."
exit 0
fi

cov combine --keep --data-file=.coverage "${segments[@]}"
cov html -d htmlcov
{
echo
echo "All segments combined: **$(cov report --format=total)%**"
echo
echo "<details><summary>Per-file report</summary>"
echo
cov report --format=markdown --skip-covered --sort=cover
echo
echo "</details>"
} >> "$GITHUB_STEP_SUMMARY"

- name: Upload HTML report
if: ${{ hashFiles('htmlcov/index.html') != '' }}
uses: actions/upload-artifact@v7
with:
name: coverage-html
overwrite: true # a re-run attempt replaces its earlier upload
path: htmlcov
retention-days: 14

# One upload per segment, each under its own flag. Codecov outages and
# tokenless refusals (Dependabot PRs) must never fail CI, hence
# fail_ci_if_error: false.
- name: Upload core coverage to Codecov
if: ${{ steps.combine.outputs.core == 'true' }}
uses: codecov/codecov-action@v7
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: coverage-core.xml
flags: core
disable_search: true
fail_ci_if_error: false

- name: Upload service coverage to Codecov
if: ${{ steps.combine.outputs.service == 'true' }}
uses: codecov/codecov-action@v7
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: coverage-service.xml
flags: service
disable_search: true
fail_ci_if_error: false

- name: Upload cluster coverage to Codecov
if: ${{ steps.combine.outputs.cluster == 'true' }}
uses: codecov/codecov-action@v7
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: coverage-cluster.xml
flags: cluster
disable_search: true
fail_ci_if_error: false

cluster-tests:

Check warning

Code scanning / CodeQL

Workflow does not contain permissions Medium test

Actions job or workflow does not limit the permissions of the GITHUB_TOKEN. Consider setting an explicit permissions block, using the following as a minimal starting point: {contents: read}
name: Cluster Tests - redis-py ${{ matrix.redis-py-version }}
# Nightly and on demand only, to keep the cluster's setup time off PRs.
# No other job passes `--run-cluster-tests`, so this is the only one that
Expand Down Expand Up @@ -218,7 +380,19 @@
# -n 0: the cluster publishes fixed ports 7001-7006, so per-worker
# clusters under xdist would collide.
run: |
uv run python -m pytest -n 0 --log-level=CRITICAL --run-cluster-tests -m requires_cluster tests/
leg="cluster-redis-py-${REDIS_PY_VERSION}"
echo "COVERAGE_LEG=$leg" >> "$GITHUB_ENV"
COVERAGE_FILE=".coverage.$leg" uv run python -m pytest -n 0 --log-level=CRITICAL --run-cluster-tests -m requires_cluster --cov --cov-report= tests/

- name: Upload coverage data
if: ${{ !cancelled() && env.COVERAGE_LEG != '' }}
uses: actions/upload-artifact@v7
with:
name: coverage-data-${{ env.COVERAGE_LEG }}
path: .coverage.${{ env.COVERAGE_LEG }}
include-hidden-files: true # the data file is a dotfile
overwrite: true # a re-run attempt replaces its earlier upload
retention-days: 7

notebooks:
name: Notebook Validation
Expand Down
2 changes: 2 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ RedisVL uses `pytest` with `testcontainers` for testing.
- `pytest --run-api-tests` - explicitly run API-dependent tests (e.g., LangCache,
external vectorizer/reranker providers). These require the appropriate API
keys and environment variables to be set.
- Coverage: `make test ARGS="--cov"`. CI uploads it to Codecov per segment (flags `core`,
`service` and `cluster`); see "Coverage in CI" in CONTRIBUTING.md. It is reported, not enforced.

## Project Structure

Expand Down
8 changes: 6 additions & 2 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -174,12 +174,16 @@ uv run pytest tests/unit/test_fields.py -v
# Run the Redis Cluster tests, which are skipped by default
uv run pytest --run-cluster-tests -m requires_cluster

# Run tests with coverage
uv run pytest --cov=redisvl --cov-report=html
# Run tests with coverage (writes an HTML report to htmlcov/)
make test ARGS="--cov --cov-report=html"
```

**Note:** Tests requiring external APIs need appropriate API keys set as environment variables.

#### Coverage in CI

CI measures coverage in segments and uploads each to Codecov under its own flag: `core` (every leg of the pull-request test matrix, which skips the tests that need API keys), `service` (the full suite including those tests, run by `make test-all` on Python 3.11) and `cluster` (the Redis Cluster tests, on the nightly schedule and manual runs only). Only the nightly run and manual (`workflow_dispatch`) runs measure every segment; on a run that skips `service` or `cluster`, Codecov carries that flag forward from the nearest ancestor commit with a report for it. Fork pull requests skip the service job. Because `service` covers nearly the whole suite, the headline number on a fork pull request can hide a drop; read the `core` row in the Codecov comment for the change itself. Dependabot pull requests get no Codecov token, so they aren't reported to Codecov. Either way, the `Coverage` job's summary on each workflow run shows the totals per segment. Coverage is reported, not enforced.

Where such an API is a *shared, stateful* service -- one managed instance reached by
every xdist worker and every concurrent CI run -- tests must namespace everything
they write and must never issue a whole-service flush, which would delete data
Expand Down
37 changes: 37 additions & 0 deletions codecov.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Coverage is reported, not enforced: both statuses are informational until
# we have enough history to choose a target.
#
# Each CI segment uploads under its own flag (see the `coverage` job in
# .github/workflows/test.yml):
# core every leg of the PR test matrix; skips the API-key tests
# service the full suite plus the API-key tests (`make test-all` on
# Python 3.11); fork PRs skip it, and Dependabot PRs upload
# nothing because they get no token
# cluster the `requires_cluster` tests, on the nightly schedule and manual
# runs only
# `service` and `cluster` are carried forward (individual_flags below): on a
# run that skips one, Codecov copies that flag from the nearest ancestor
# commit with a report for it. A new segment that PRs can skip needs its own
# entry there, plus a loop entry and an upload step in the `coverage` job in
# .github/workflows/test.yml.
coverage:
status:
project:
default:
informational: true
patch:
default:
informational: true

flag_management:
default_rules:
carryforward: false
individual_flags:
- name: service
carryforward: true
- name: cluster
carryforward: true

comment:
layout: "condensed_header, diff, flags"
require_changes: true
23 changes: 23 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,7 @@ dev = [
"pylint>=3.1.0,<5",
"pytest>=9.0.3,<10",
"pytest-asyncio>=1,<2",
"pytest-cov>=7,<8",
"pytest-xdist[psutil]>=3.6.1,<4",
"pre-commit>=4.1.0,<5",
"mypy>=1.11.0,<2",
Expand Down Expand Up @@ -190,6 +191,28 @@ log_cli = true
asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "function"

[tool.coverage.run]
# source_pkgs, not source: the XML report writes paths relative to `source`,
# so `redisvl/version.py` would appear as a bare `version.py` that Codecov
# can't tell apart from `redisvl/cli/version.py`. With source_pkgs every path
# is repo-relative (`redisvl/...`).
source_pkgs = ["redisvl"]
branch = true
# Paths are stored relative to the checkout, so data from different CI
# runners combines into one report.
relative_files = true
# The CLI tests run `python -m redisvl.cli.runner` in a subprocess; without
# this, the CLI is measured as untested.
patch = ["subprocess"]

[tool.coverage.report]
exclude_also = [
"if TYPE_CHECKING:",
"@overload",
"raise NotImplementedError",
"^\\s*\\.\\.\\.$",
]

[tool.mypy]
warn_unused_configs = true
ignore_missing_imports = true
Expand Down
Loading
Loading