Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@

All notable changes to this project are documented in this file.

## Unreleased

### Fixed
- **`number_of_cases` is now emitted as a JSON integer instead of source text.** The slot is typed `int` on biolink-model's `EntityToDiseaseAssociation` / `EntityToPhenotypicFeatureAssociation`, but it was absent from `lib.numeric_columns`' exact set — since 15.1's `STUDY_SIZE_EXEMPT_PATTERN` (#119) stopped it being *renamed* onto `study_size`, nothing cast its *values*, so the raw TSV cell (`"1"`) shipped on the edge NDJSON. Pydantic's lax validation coerced the string back to int inside `validate_kgx`, so the record still validated, and the Rust `uuid_on_collision: merge` recompute already wrote a real int — leaving the shipped graph type-inconsistent edge to edge. `number_of_cases` now rides the same `clean_numeric` / `format_numeric` machinery as `study_size`: `numeric_slot_kind` reads the `int` typing off the installed model, fractional / negative / non-numeric counts become null, and the release-mode `drop_low_number_of_cases` filter is untouched (it already cast inline).

## 16.6.1 - 2026-09-04

### Performance
Expand Down
19 changes: 10 additions & 9 deletions src/tablassert/lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -698,27 +698,27 @@ def math_op(lf: pl.LazyFrame, col: str, func: Functions, args: list[Literal[Toke
def numeric_columns(names: list[str]) -> list[str]:
"""Return column names that should be coerced and formatted as numbers.

P-value columns by substring plus the exact ``effect_size`` and
``study_size`` fields. The old ``sample_size`` / ``relationship_strength`` /
``supporting_study_size`` names are absent on purpose: the column coercions
rename them to ``study_size`` / ``effect_size`` before ``clean_numeric`` /
``format_numeric`` run.
P-value columns by substring plus the exact ``effect_size``,
``number_of_cases`` and ``study_size`` fields. The old ``sample_size`` /
``relationship_strength`` / ``supporting_study_size`` names are absent on
purpose: the column coercions rename them to ``study_size`` / ``effect_size``
before ``clean_numeric`` / ``format_numeric`` run.

Args:
names: Schema column names to filter.

Returns:
Subset of ``names`` destined for numeric coercion/formatting.
"""
# P-value columns by substring plus exact effect-size and study-size fields.
exact: set[str] = {"effect_size", "study_size"}
# P-value columns by substring plus exact effect-size, case-count and study-size fields.
exact: set[str] = {"effect_size", "number_of_cases", "study_size"}
return [c for c in names if ("p_value" in c.lower()) or (c in exact)]


def clean_numeric(lf: pl.LazyFrame) -> pl.LazyFrame:
"""Coerce numeric annotation columns to Float64, dropping non-numeric values to null.

Only touches p-value, effect-size and study-size columns.
Only touches p-value, effect-size, number-of-cases and study-size columns.

Args:
lf: Source LazyFrame.
Expand Down Expand Up @@ -746,7 +746,8 @@ def format_numeric(lf: pl.LazyFrame) -> pl.LazyFrame:

Remaining numeric columns are emitted as real JSON numbers when the installed
biolink model types them ``int`` / ``float`` -- ``effect_size`` (``float``,
biolink-model#1774) and ``study_size`` (``int`` on the inlined ``Study``,
biolink-model#1774), ``number_of_cases`` (``int`` on the disease/phenotype
``Association`` classes) and ``study_size`` (``int`` on the inlined ``Study``,
biolink-model#1770) -- and keep the controlled decimal general string notation
(``{:.4g}``) only while a slot stays untyped, since such values end up in
human-readable text. Null values stay null.
Expand Down
19 changes: 17 additions & 2 deletions tests/test_lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -1301,10 +1301,11 @@ def test_numeric_columns_matches_p_value_substring() -> None:


def test_numeric_columns_matches_exact_names() -> None:
"""numeric_columns matches exact effect size and study size names."""
names: list[str] = ["effect_size", "study_size", "cohort", "sample_size", "relationship_strength"]
"""numeric_columns matches exact effect size, number-of-cases and study size names."""
names: list[str] = ["effect_size", "number_of_cases", "study_size", "cohort", "sample_size", "relationship_strength"]
result: list[str] = numeric_columns(names)
assert "effect_size" in result
assert "number_of_cases" in result
assert "study_size" in result
assert "cohort" not in result
# Old names are superseded: coercion renames them before clean_numeric/format_numeric run.
Expand Down Expand Up @@ -1406,6 +1407,20 @@ def test_format_numeric_nulls_invalid_study_counts() -> None:
assert result.schema["study_size"] == pl.Int64


def test_format_numeric_emits_number_of_cases_as_int() -> None:
"""number_of_cases leaves the pipeline as a real JSON int, not a TSV-text string.

biolink-model 4.4.4 types ``number_of_cases`` ``int`` on the disease/phenotype
``Association`` classes, and the Rust ``uuid_on_collision: merge`` recompute already
writes an int union length, so first-wins edges must not ship the raw source string.
Fractional, negative, non-finite and non-numeric counts become null, like study counts.
"""
lf: pl.LazyFrame = pl.DataFrame({"number_of_cases": ["1", "25", "0.42", "-3", "abc", None]}).lazy()
result: pl.DataFrame = format_numeric(clean_numeric(lf)).collect()
assert result["number_of_cases"].to_list() == [1, 25, None, None, None, None]
assert result.schema["number_of_cases"] == pl.Int64


def test_format_numeric_preserves_nulls() -> None:
"""format_numeric preserves nulls as null."""
lf: pl.LazyFrame = pl.DataFrame({"p_value": ["1e-8", "N/A", "0.05"]}).lazy()
Expand Down
Loading