From 2c571039f33a118c49e07fb8b0d59ac5473ab5f2 Mon Sep 17 00:00:00 2001 From: yuxt Date: Thu, 12 Nov 2026 23:39:19 +0800 Subject: [PATCH 1/3] add tests --- chainladder/core/tests/test_pandas.py | 114 ++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/chainladder/core/tests/test_pandas.py b/chainladder/core/tests/test_pandas.py index 82909f50d..d53e05f95 100644 --- a/chainladder/core/tests/test_pandas.py +++ b/chainladder/core/tests/test_pandas.py @@ -196,3 +196,117 @@ def test_aggregation_default_axis_singleton_result(raa: Triangle, backend, keepd else: assert isinstance(result, np.floating) assert result == triangle.values[0, 0, 0, 0] + + +def _nan_cumsum(values: np.ndarray, axis: int) -> np.ndarray: + """Cumulative sum skipping NaN that leaves NaN cells as NaN, like pandas.""" + result = np.where(np.isnan(values), np.nan, np.nancumsum(values, axis=axis)) + result[result == 0] = np.nan + return result + + +def _nan_diff(values: np.ndarray, axis: int) -> np.ndarray: + """First difference keeping the shape with NaN in the first slot, like pandas.""" + result = np.full_like(values, np.nan) + head = [slice(None)] * values.ndim + tail = [slice(None)] * values.ndim + head[axis] = slice(1, None) + tail[axis] = slice(None, -1) + result[tuple(head)] = values[tuple(head)] - values[tuple(tail)] + result[result == 0] = np.nan + return result + + +def _assert_same_labels(result: Triangle, triangle: Triangle) -> None: + assert result.index.equals(triangle.index) + assert result.columns.equals(triangle.columns) + result_origin, expected_origin = result.origin, triangle.origin + if isinstance(expected_origin, pd.Series): + assert isinstance(result_origin, pd.Series) + pd.testing.assert_series_equal(result_origin, expected_origin) + else: + assert isinstance(result_origin, pd.PeriodIndex) + pd.testing.assert_index_equal(result_origin, expected_origin) + assert result.development.equals(triangle.development) + + +@pytest.mark.parametrize( + ("method", "expected_func"), [("cumsum", _nan_cumsum), ("diff", _nan_diff)] +) +@pytest.mark.parametrize("axis", ["index", "columns", "origin", "development"]) +def test_triangle_shape_preserving_values(clrd, method, expected_func, axis): + """ + Apply ``cumsum`` and ``diff`` along each axis without collapsing it. + + The result is a Triangle with the input's shape, labels and backend; empty + cells stay NaN and the input triangle is not modified. + """ + triangle = clrd.iloc[13:16, :2, -5:, :5] + before = triangle.copy() + values = triangle.set_backend("numpy").values + assert np.isnan(values).any() + expected = expected_func(values, triangle._get_axis(axis)) + + result = getattr(triangle, method)(axis=axis, auto_sparse=False) + + assert result.shape == triangle.shape + assert result.array_backend == triangle.array_backend + _assert_same_labels(result, triangle) + np.testing.assert_allclose( + result.set_backend("numpy").values, expected, equal_nan=True + ) + assert triangle.equals(before) + + +def test_cumsum_development_inverts_cum_to_incr(raa): + """Cumulating incremental amounts along development restores the triangle.""" + result = raa.cum_to_incr().cumsum(axis="development") + + assert result.shape == raa.shape + np.testing.assert_allclose( + result.set_backend("numpy").values, + raa.set_backend("numpy").values, + equal_nan=True, + ) + + +def test_diff_development_matches_cum_to_incr(raa): + """Differencing along development gives the increments after the first age.""" + result = raa.diff(axis="development").set_backend("numpy").values + incremental = raa.cum_to_incr().set_backend("numpy").values + + assert np.isnan(result[..., 0]).all() + np.testing.assert_allclose(result[..., 1:], incremental[..., 1:], equal_nan=True) + + +@pytest.mark.parametrize("method", ["cumsum", "diff"]) +def test_shape_preserving_default_axis_first_non_singleton(raa, method): + """With ``axis=None``, use the first axis of length > 1, like the reductions.""" + result = getattr(raa, method)(auto_sparse=False) + expected = getattr(raa, method)(axis="origin", auto_sparse=False) + + assert result.equals(expected) + + +def test_groupby_cumsum(clrd): + """ + Cumulate within each group and keep every row in the original order. + + Like pandas ``groupby(...).cumsum()``, the result has the input's shape and + index, and each row is the running total of the earlier rows in its group. + """ + triangle = clrd.iloc[:12, :2, -4:, :4] + assert not triangle.index["LOB"].is_monotonic_increasing + values = triangle.set_backend("numpy").values + expected = np.empty_like(values) + for _, rows in triangle.index.groupby("LOB").indices.items(): + expected[rows] = _nan_cumsum(values[rows], axis=0) + + result = triangle.groupby("LOB").cumsum(auto_sparse=False) + + assert result.shape == triangle.shape + assert result.array_backend == triangle.array_backend + _assert_same_labels(result, triangle) + np.testing.assert_allclose( + result.set_backend("numpy").values, expected, equal_nan=True + ) From a7a6b2942b43e5272580ca757d6d98917939d0ed Mon Sep 17 00:00:00 2001 From: yuxt Date: Fri, 13 Nov 2026 00:16:18 +0800 Subject: [PATCH 2/3] fixes --- chainladder/core/pandas.py | 177 +++++++++++++++++++- chainladder/core/tests/test_pandas.py | 232 ++++++++++++++++++++++++-- 2 files changed, 388 insertions(+), 21 deletions(-) diff --git a/chainladder/core/pandas.py b/chainladder/core/pandas.py index ecab025f2..8c4336979 100644 --- a/chainladder/core/pandas.py +++ b/chainladder/core/pandas.py @@ -7,6 +7,7 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. from __future__ import annotations +import operator import warnings from typing import TYPE_CHECKING, cast @@ -98,9 +99,53 @@ def std(self, *args: Any, **kwargs: Any) -> Triangle: """Refer to pandas for ``std`` functionality.""" return _groupby_aggregate(self, "std", *args, **kwargs) - def cumsum(self, *args: Any, **kwargs: Any) -> Triangle: - """Refer to pandas for ``cumsum`` functionality.""" - return _groupby_aggregate(self, "cumsum", *args, **kwargs) + def cumsum(self, *, skipna: bool = True, auto_sparse: bool = True) -> Triangle: + """ + Cumulate within each group, keeping every label in its original order. + + Unlike the other groupby methods, this does not reduce each group to a + single label. Like pandas ``groupby(...).cumsum()``, the result has the + shape and labels of the grouped Triangle. + + Parameters + ---------- + skipna : bool, default True + Passed to :meth:`~chainladder.Triangle.cumsum`. + auto_sparse : bool, default True + Whether the result's array backend may be switched based on its size + and density. ``False`` keeps the input's backend. + + Returns + ------- + Triangle + Running totals within each group. Labels with a missing group key + are NaN. + """ + obj = self.obj.copy() + xp = obj.get_array_module() + values = [] + positions = [] + for rows in self.groups.indices.values(): + indexer = [slice(None)] * self.axis + [rows] + group = obj.iloc[tuple(indexer)].cumsum( + self.axis, skipna=skipna, auto_sparse=False + ) + values.append(group.values) + positions.extend(rows) + + # Rows excluded by groupby (e.g. missing keys) have no running total. + missing = np.setdiff1d(np.arange(obj.shape[self.axis]), positions) + if len(missing): + shape = list(obj.shape) + shape[self.axis] = len(missing) + values.append(xp.full(tuple(shape), xp.nan)) + positions.extend(missing) + if values: + indexer = [slice(None)] * self.axis + [np.argsort(positions)] + obj.values = xp.concatenate(values, axis=self.axis)[tuple(indexer)] + if auto_sparse: + obj = obj._auto_sparse() + return obj def quantile(self, *args: Any, **kwargs: Any) -> Triangle: """Refer to pandas for ``quantile`` functionality.""" @@ -159,10 +204,64 @@ def std(self, axis: str | int | None = None, *args: Any, **kwargs: Any) -> Trian return _triangle_aggregate(self, "nanstd", axis, *args, **kwargs) def cumsum( - self, axis: str | int | None = None, *args: Any, **kwargs: Any + self, + axis: str | int | None = None, + *, + skipna: bool = True, + auto_sparse: bool = True, ) -> Triangle: - """Refer to pandas for ``cumsum`` functionality.""" - return _triangle_aggregate(self, "nancumsum", axis, *args, **kwargs) + """ + Cumulate values along an axis, keeping the shape and labels. + + Only the values change. Flags such as ``is_cumulative`` are left as they + are; use :meth:`~chainladder.Triangle.incr_to_cum` to convert an + incremental Triangle. + + Parameters + ---------- + axis : str | int | None + Axis to cumulate along. ``None`` uses the first axis whose length is + not 1, or the index if every axis has length 1. + skipna : bool, default True + If True, missing cells are skipped and the running total continues + after them. If False, every cell after a missing one is missing. + auto_sparse : bool, default True + Whether the result's array backend may be switched based on its size + and density. ``False`` keeps the input's backend. + + Returns + ------- + Triangle + Running totals with the input's shape and labels. Missing cells stay + missing, and zero totals are stored as NaN. + """ + obj = self.copy() + axis = ( + next((i for i, size in enumerate(obj.shape) if size != 1), 0) + if axis is None + else self._get_axis(axis) + ) + xp = obj.get_array_module() + if obj.array_backend == "sparse": + # sparse has no cumsum: add one slice at a time, which stays sparse. + func = xp.nansum if skipna else xp.sum + prefixes = [] + running = None + for stop in range(1, obj.shape[axis] + 1): + indexer = [slice(None)] * axis + [slice(stop - 1, stop)] + increment = func(obj.values[tuple(indexer)], axis, keepdims=True) + running = increment if running is None else running + increment + prefixes.append(running) + values = xp.concatenate(prefixes, axis=axis) if prefixes else obj.values + else: + func = xp.nancumsum if skipna else xp.cumsum + values = func(obj.values, axis) + obj.values = cast( + "BackendArray", num_to_nan(xp.where(xp.isnan(obj.values), xp.nan, values)) + ) + if auto_sparse: + obj = obj._auto_sparse() + return obj def quantile( self, axis: str | int | None = None, *args: Any, **kwargs: Any @@ -171,10 +270,70 @@ def quantile( return _triangle_aggregate(self, "nanquantile", axis, *args, **kwargs) def diff( - self, axis: str | int | None = None, *args: Any, **kwargs: Any + self, + axis: str | int | None = None, + periods: int = 1, + *, + auto_sparse: bool = True, ) -> Triangle: - """Refer to pandas for ``diff`` functionality.""" - return _triangle_aggregate(self, "diff", axis, *args, **kwargs) + """ + Difference values along an axis, keeping the shape and labels. + + Each cell has the cell ``periods`` positions away subtracted from it, + like pandas ``diff``. Unlike pandas, ``axis`` is the first argument. + Only the values change. Flags such as ``is_cumulative`` are left as they + are; use :meth:`~chainladder.Triangle.cum_to_incr` to convert a + cumulative Triangle. + + Parameters + ---------- + axis : str | int | None + Axis to difference along. ``None`` uses the first axis whose length + is not 1, or the index if every axis has length 1. + periods : int, default 1 + Distance to the cell that is subtracted. Positive values subtract an + earlier cell, negative values a later one. + auto_sparse : bool, default True + Whether the result's array backend may be switched based on its size + and density. ``False`` keeps the input's backend. + + Returns + ------- + Triangle + Differences with the input's shape and labels. Cells with no cell to + subtract are NaN, and zero differences are stored as NaN. + """ + periods = operator.index(periods) + obj = self.copy() + axis = ( + next((i for i, size in enumerate(obj.shape) if size != 1), 0) + if axis is None + else self._get_axis(axis) + ) + xp = obj.get_array_module() + if periods == 0 or abs(periods) >= obj.shape[axis]: + # Every difference is zero, or no cell has one to subtract. + obj.values = xp.full(obj.shape, xp.nan) + else: + current = [slice(None)] * 4 + previous = [slice(None)] * 4 + current[axis] = ( + slice(periods, None) if periods > 0 else slice(None, periods) + ) + previous[axis] = ( + slice(None, -periods) if periods > 0 else slice(-periods, None) + ) + values = obj.values[tuple(current)] - obj.values[tuple(previous)] + shape = list(obj.shape) + shape[axis] = abs(periods) + padding = xp.full(tuple(shape), xp.nan) + parts = [padding, values] if periods > 0 else [values, padding] + obj.values = cast( + "BackendArray", num_to_nan(xp.concatenate(parts, axis=axis)) + ) + if auto_sparse: + obj = obj._auto_sparse() + return obj def to_clipboard(self, *args: Any, **kwargs: Any) -> None: """Refer to pandas for ``to_clipboard`` functionality.""" diff --git a/chainladder/core/tests/test_pandas.py b/chainladder/core/tests/test_pandas.py index d53e05f95..f095765f3 100644 --- a/chainladder/core/tests/test_pandas.py +++ b/chainladder/core/tests/test_pandas.py @@ -23,7 +23,6 @@ "prod", "var", "std", - "cumsum", "quantile", ] PASSTHRU_METHODS = [ @@ -63,26 +62,26 @@ def test_pandas_methods_are_defined_on_classes(): This lets static type checkers such as Pyright/Pylance and ty see them. """ assert ( - set(AGGREGATIONS + ["diff"] + PASSTHRU_METHODS) + set(AGGREGATIONS + ["cumsum", "diff"] + PASSTHRU_METHODS) <= TrianglePandas.__dict__.keys() ) - assert set(AGGREGATIONS) <= TriangleGroupBy.__dict__.keys() + assert set(AGGREGATIONS + ["cumsum"]) <= TriangleGroupBy.__dict__.keys() -@pytest.mark.parametrize("method", AGGREGATIONS + ["diff"]) +@pytest.mark.parametrize("method", AGGREGATIONS) def test_triangle_aggregation_dispatch(raa, monkeypatch, method): """ Delegate each Triangle method to ``_triangle_aggregate`` unchanged. - Aggregations pass the ``nan``-prefixed NumPy function name (e.g. ``nansum``); - ``diff`` passes ``diff``. Arguments and the return value pass through as is. + Aggregations pass the ``nan``-prefixed NumPy function name (e.g. ``nansum``). + Arguments and the return value pass through as is. """ aggregate = Mock() monkeypatch.setattr(triangle_pandas, "_triangle_aggregate", aggregate) result = getattr(raa, method)("origin", "arg", keepdims=True, auto_sparse=False) - backend_method = method if method == "diff" else "nan" + method + backend_method = "nan" + method aggregate.assert_called_once_with( raa, backend_method, "origin", "arg", keepdims=True, auto_sparse=False ) @@ -125,7 +124,7 @@ def test_dataframe_passthru_dispatch(raa, monkeypatch, method): assert result is passthru.return_value -@pytest.mark.parametrize("method", [name for name in AGGREGATIONS if name != "cumsum"]) +@pytest.mark.parametrize("method", AGGREGATIONS) @pytest.mark.parametrize("axis", ["index", "columns", "origin", "development"]) @pytest.mark.filterwarnings("error::RuntimeWarning") def test_triangle_aggregation_values(clrd, method, axis): @@ -280,14 +279,144 @@ def test_diff_development_matches_cum_to_incr(raa): @pytest.mark.parametrize("method", ["cumsum", "diff"]) -def test_shape_preserving_default_axis_first_non_singleton(raa, method): - """With ``axis=None``, use the first axis of length > 1, like the reductions.""" - result = getattr(raa, method)(auto_sparse=False) - expected = getattr(raa, method)(axis="origin", auto_sparse=False) +@pytest.mark.parametrize("axis", range(4)) +def test_shape_preserving_default_axis_first_non_singleton(clrd, method, axis): + """With ``axis=None``, use the first non-singleton axis, like the reductions.""" + indexer = [slice(11, 13), slice(0, 2), slice(0, 5), slice(0, 5)] + for preceding_axis in range(axis): + start = indexer[preceding_axis].start + indexer[preceding_axis] = slice(start, start + 1) + triangle = clrd.iloc[tuple(indexer)] + result = getattr(triangle, method)(auto_sparse=False) + expected = getattr(triangle, method)(axis=axis, auto_sparse=False) + + assert result.equals(expected) + + +@pytest.mark.parametrize("method", ["cumsum", "diff"]) +def test_shape_preserving_singleton(raa, method): + """ + A (1, 1, 1, 1) Triangle stays a Triangle instead of becoming a scalar. + ``cumsum`` keeps its value and ``diff`` is NaN. + """ + triangle = raa.iloc[:, :, :1, :1] + result = getattr(triangle, method)(auto_sparse=False) + + assert result.shape == triangle.shape + assert result.array_backend == triangle.array_backend + _assert_same_labels(result, triangle) + values = result.set_backend("numpy").values + if method == "diff": + assert np.isnan(values).all() + else: + np.testing.assert_allclose(values, triangle.set_backend("numpy").values) + + +@pytest.mark.parametrize("method", ["cumsum", "diff"]) +def test_shape_preserving_negative_axis(raa, method): + """Negative axis numbers count back from the development axis.""" + result = getattr(raa, method)(axis=-1, auto_sparse=False) + expected = getattr(raa, method)(axis="development", auto_sparse=False) assert result.equals(expected) +@pytest.mark.parametrize("method", ["cumsum", "diff"]) +def test_shape_preserving_empty_triangle(raa, method): + """A Triangle with no index rows is returned empty, on the same backend.""" + triangle = raa.iloc[:0, :, :, :] + + result = getattr(triangle, method)(auto_sparse=False) + + assert result.shape == triangle.shape == (0, 1, 10, 10) + assert result.array_backend == triangle.array_backend + + +@pytest.mark.parametrize("method", ["cumsum", "diff"]) +def test_shape_preserving_auto_sparse(raa, method): + """ + By default a small sparse result is converted to numpy, like the reductions. + + ``auto_sparse=False`` keeps it sparse, with the same values. + """ + triangle = raa.set_backend("sparse") + + default = getattr(triangle, method)(axis="development") + kept = getattr(triangle, method)(axis="development", auto_sparse=False) + + assert default.array_backend == "numpy" + assert kept.array_backend == "sparse" + assert default.equals(kept) + + +@pytest.mark.parametrize("skipna", [True, False]) +def test_cumsum_internal_missing_cells(raa, skipna): + """ + Continue the total after a missing cell only with ``skipna=True``. + + The missing cell stays missing either way, and a zero total is stored as NaN. + """ + triangle = raa.iloc[:, :, :1, :5] + triangle.values = triangle.get_array_module().array( + np.array([2.0, np.nan, 3.0, -5.0, 4.0]).reshape(triangle.shape) + ) + before = triangle.copy() + result = triangle.cumsum(axis="development", skipna=skipna, auto_sparse=False) + expected = [2.0, np.nan, 5.0, np.nan, 4.0] if skipna else [2.0] + [np.nan] * 4 + + np.testing.assert_allclose( + result.set_backend("numpy").values.ravel(), expected, equal_nan=True + ) + assert triangle.equals(before) + + +@pytest.mark.parametrize("axis", range(4)) +@pytest.mark.parametrize("periods", [0, 2, -1, -2, 10, 12]) +def test_diff_periods(clrd, axis, periods): + """Match pandas ``diff`` for each axis and ``periods``, with zeros as NaN.""" + triangle = clrd.iloc[13:16, :2, -5:, :5] + values = np.moveaxis(triangle.set_backend("numpy").values, axis, 0) + expected = ( + pd + .DataFrame(values.reshape(values.shape[0], -1)) + .diff(periods=periods) + .to_numpy() + .reshape(values.shape) + ) + expected = np.moveaxis(expected, 0, axis) + expected[expected == 0] = np.nan + + result = triangle.diff(axis=axis, periods=periods, auto_sparse=False) + + assert result.shape == triangle.shape + assert result.array_backend == triangle.array_backend + _assert_same_labels(result, triangle) + np.testing.assert_allclose( + result.set_backend("numpy").values, expected, equal_nan=True + ) + + +def test_diff_periods_must_be_integer(raa): + """A non-integer ``periods`` raises instead of being truncated.""" + with pytest.raises(TypeError): + raa.diff(axis="development", periods=1.5) + + +def test_sparse_transforms_do_not_densify(raa, monkeypatch): + """``cumsum`` and ``diff`` on a sparse Triangle never call ``todense``.""" + triangle = raa.set_backend("sparse") + + def reject_densification(*_args, **_kwargs): + pytest.fail("Transform unexpectedly densified its sparse input") + + with monkeypatch.context() as patch: + patch.setattr(type(triangle.values), "todense", reject_densification) + cumulative = triangle.cumsum(axis="development", auto_sparse=False) + difference = triangle.diff(axis="development", auto_sparse=False) + + assert cumulative.array_backend == difference.array_backend == "sparse" + + def test_groupby_cumsum(clrd): """ Cumulate within each group and keep every row in the original order. @@ -296,6 +425,7 @@ def test_groupby_cumsum(clrd): index, and each row is the running total of the earlier rows in its group. """ triangle = clrd.iloc[:12, :2, -4:, :4] + before = triangle.copy() assert not triangle.index["LOB"].is_monotonic_increasing values = triangle.set_backend("numpy").values expected = np.empty_like(values) @@ -310,3 +440,81 @@ def test_groupby_cumsum(clrd): np.testing.assert_allclose( result.set_backend("numpy").values, expected, equal_nan=True ) + assert triangle.equals(before) + + +@pytest.mark.parametrize("axis", [1, 2, 3]) +def test_groupby_cumsum_other_axes(clrd, axis): + """Group along the columns, origin and development axes as well.""" + triangle = clrd.iloc[13:16, :4, -4:, :4] + groups = ["a", "b", "a", "b"] + values = triangle.set_backend("numpy").values + expected = np.empty_like(values) + for positions in ([0, 2], [1, 3]): + indexer = [slice(None)] * axis + [positions] + expected[tuple(indexer)] = _nan_cumsum(values[tuple(indexer)], axis) + + result = triangle.groupby(groups, axis=axis).cumsum(auto_sparse=False) + + assert result.shape == triangle.shape + assert result.array_backend == triangle.array_backend + _assert_same_labels(result, triangle) + np.testing.assert_allclose( + result.set_backend("numpy").values, expected, equal_nan=True + ) + + +@pytest.mark.parametrize("all_missing", [True, False]) +def test_groupby_cumsum_missing_keys(clrd, all_missing): + """Labels with a missing group key are NaN; the other groups still cumulate.""" + triangle = clrd.iloc[:12, :2, -4:, :4] + labels = triangle.index.copy() + labels.loc[labels.index if all_missing else [0], "LOB"] = None + triangle.index = labels + values = triangle.set_backend("numpy").values + expected = np.full_like(values, np.nan) + for rows in labels.groupby("LOB").indices.values(): + expected[rows] = _nan_cumsum(values[rows], axis=0) + + result = triangle.groupby("LOB").cumsum(auto_sparse=False) + + assert result.shape == triangle.shape + assert result.array_backend == triangle.array_backend + _assert_same_labels(result, triangle) + np.testing.assert_allclose( + result.set_backend("numpy").values, expected, equal_nan=True + ) + + +def test_groupby_cumsum_auto_sparse(clrd): + """ + By default a small sparse result is converted to numpy, like the reductions. + + ``auto_sparse=False`` keeps it sparse, with the same values. + """ + triangle = clrd.iloc[:12, :2, -4:, :4].set_backend("sparse") + + default = triangle.groupby("LOB").cumsum() + kept = triangle.groupby("LOB").cumsum(auto_sparse=False) + + assert default.array_backend == "numpy" + assert kept.array_backend == "sparse" + assert default.equals(kept) + + +@pytest.mark.parametrize("skipna", [True, False]) +def test_groupby_cumsum_skipna(raa, skipna): + """``skipna`` applies within each group and does not carry across groups.""" + triangle = raa.iloc[:, :, :1, :4] + triangle.values = triangle.get_array_module().array( + np.array([2.0, np.nan, 3.0, 4.0]).reshape(triangle.shape) + ) + expected = [2.0, np.nan, 5.0, 4.0] if skipna else [2.0, np.nan, np.nan, 4.0] + + result = triangle.groupby(["a", "a", "a", "b"], axis=3).cumsum( + skipna=skipna, auto_sparse=False + ) + + np.testing.assert_allclose( + result.set_backend("numpy").values.ravel(), expected, equal_nan=True + ) From d883d2507b09151d33a590957ec4d3232c9c9b7a Mon Sep 17 00:00:00 2001 From: yuxt Date: Fri, 13 Nov 2026 00:31:44 +0800 Subject: [PATCH 3/3] fixes --- chainladder/core/pandas.py | 51 ++--- chainladder/core/tests/test_pandas.py | 264 ++++++++++++++++---------- 2 files changed, 188 insertions(+), 127 deletions(-) diff --git a/chainladder/core/pandas.py b/chainladder/core/pandas.py index 8c4336979..a2f828b48 100644 --- a/chainladder/core/pandas.py +++ b/chainladder/core/pandas.py @@ -99,7 +99,7 @@ def std(self, *args: Any, **kwargs: Any) -> Triangle: """Refer to pandas for ``std`` functionality.""" return _groupby_aggregate(self, "std", *args, **kwargs) - def cumsum(self, *, skipna: bool = True, auto_sparse: bool = True) -> Triangle: + def cumsum(self, *, auto_sparse: bool = True) -> Triangle: """ Cumulate within each group, keeping every label in its original order. @@ -109,8 +109,6 @@ def cumsum(self, *, skipna: bool = True, auto_sparse: bool = True) -> Triangle: Parameters ---------- - skipna : bool, default True - Passed to :meth:`~chainladder.Triangle.cumsum`. auto_sparse : bool, default True Whether the result's array backend may be switched based on its size and density. ``False`` keeps the input's backend. @@ -118,8 +116,9 @@ def cumsum(self, *, skipna: bool = True, auto_sparse: bool = True) -> Triangle: Returns ------- Triangle - Running totals within each group. Labels with a missing group key - are NaN. + Running totals within each group, following + :meth:`~chainladder.Triangle.cumsum`. Labels with a missing group + key are NaN. """ obj = self.obj.copy() xp = obj.get_array_module() @@ -127,9 +126,7 @@ def cumsum(self, *, skipna: bool = True, auto_sparse: bool = True) -> Triangle: positions = [] for rows in self.groups.indices.values(): indexer = [slice(None)] * self.axis + [rows] - group = obj.iloc[tuple(indexer)].cumsum( - self.axis, skipna=skipna, auto_sparse=False - ) + group = obj.iloc[tuple(indexer)].cumsum(self.axis, auto_sparse=False) values.append(group.values) positions.extend(rows) @@ -207,12 +204,16 @@ def cumsum( self, axis: str | int | None = None, *, - skipna: bool = True, auto_sparse: bool = True, ) -> Triangle: """ Cumulate values along an axis, keeping the shape and labels. + Triangles store zeros as NaN, so a NaN inside the triangle counts as + zero and the running total carries through it, as in + :meth:`~chainladder.Triangle.incr_to_cum`. Only cells outside the + triangle, which are valued after ``valuation_date``, stay NaN. + Only the values change. Flags such as ``is_cumulative`` are left as they are; use :meth:`~chainladder.Triangle.incr_to_cum` to convert an incremental Triangle. @@ -222,9 +223,6 @@ def cumsum( axis : str | int | None Axis to cumulate along. ``None`` uses the first axis whose length is not 1, or the index if every axis has length 1. - skipna : bool, default True - If True, missing cells are skipped and the running total continues - after them. If False, every cell after a missing one is missing. auto_sparse : bool, default True Whether the result's array backend may be switched based on its size and density. ``False`` keeps the input's backend. @@ -232,8 +230,8 @@ def cumsum( Returns ------- Triangle - Running totals with the input's shape and labels. Missing cells stay - missing, and zero totals are stored as NaN. + Running totals with the input's shape and labels. Zero totals are + stored as NaN. """ obj = self.copy() axis = ( @@ -244,20 +242,18 @@ def cumsum( xp = obj.get_array_module() if obj.array_backend == "sparse": # sparse has no cumsum: add one slice at a time, which stays sparse. - func = xp.nansum if skipna else xp.sum prefixes = [] running = None for stop in range(1, obj.shape[axis] + 1): indexer = [slice(None)] * axis + [slice(stop - 1, stop)] - increment = func(obj.values[tuple(indexer)], axis, keepdims=True) + increment = xp.nan_to_num(obj.values[tuple(indexer)]) running = increment if running is None else running + increment prefixes.append(running) values = xp.concatenate(prefixes, axis=axis) if prefixes else obj.values else: - func = xp.nancumsum if skipna else xp.cumsum - values = func(obj.values, axis) + values = xp.nancumsum(obj.values, axis) obj.values = cast( - "BackendArray", num_to_nan(xp.where(xp.isnan(obj.values), xp.nan, values)) + "BackendArray", num_to_nan(values * obj.nan_triangle[None, None]) ) if auto_sparse: obj = obj._auto_sparse() @@ -281,6 +277,12 @@ def diff( Each cell has the cell ``periods`` positions away subtracted from it, like pandas ``diff``. Unlike pandas, ``axis`` is the first argument. + + Triangles store zeros as NaN, so a NaN inside the triangle counts as + zero, as in :meth:`~chainladder.Triangle.cum_to_incr`. A difference is + NaN only when either of its two cells is outside the triangle, that is, + valued after ``valuation_date``, or when there is no cell to subtract. + Only the values change. Flags such as ``is_cumulative`` are left as they are; use :meth:`~chainladder.Triangle.cum_to_incr` to convert a cumulative Triangle. @@ -300,8 +302,8 @@ def diff( Returns ------- Triangle - Differences with the input's shape and labels. Cells with no cell to - subtract are NaN, and zero differences are stored as NaN. + Differences with the input's shape and labels. Zero differences are + stored as NaN. """ periods = operator.index(periods) obj = self.copy() @@ -323,7 +325,12 @@ def diff( previous[axis] = ( slice(None, -periods) if periods > 0 else slice(-periods, None) ) - values = obj.values[tuple(current)] - obj.values[tuple(previous)] + values = xp.where(xp.isnan(obj.values), 0, obj.values) + values = values[tuple(current)] - values[tuple(previous)] + mask = obj.nan_triangle[None, None] + if axis in (2, 3): + mask = mask[tuple(current)] * mask[tuple(previous)] + values = values * mask shape = list(obj.shape) shape[axis] = abs(periods) padding = xp.full(tuple(shape), xp.nan) diff --git a/chainladder/core/tests/test_pandas.py b/chainladder/core/tests/test_pandas.py index f095765f3..8eaef52c9 100644 --- a/chainladder/core/tests/test_pandas.py +++ b/chainladder/core/tests/test_pandas.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Literal from unittest.mock import Mock import numpy as np @@ -197,25 +197,57 @@ def test_aggregation_default_axis_singleton_result(raa: Triangle, backend, keepd assert result == triangle.values[0, 0, 0, 0] -def _nan_cumsum(values: np.ndarray, axis: int) -> np.ndarray: - """Cumulative sum skipping NaN that leaves NaN cells as NaN, like pandas.""" - result = np.where(np.isnan(values), np.nan, np.nancumsum(values, axis=axis)) +AXES = ["index", "columns", "origin", "development"] + + +def _numpy_values(triangle: Triangle) -> np.ndarray: + """The triangle's values as a numpy array, whatever its backend.""" + values = triangle.set_backend("numpy").values + assert isinstance(values, np.ndarray) + return values + + +def _triangle_mask(triangle: Triangle) -> np.ndarray: + """``nan_triangle`` as a numpy array: 1 inside the triangle, NaN outside it.""" + return np.asarray(triangle.set_backend("numpy").nan_triangle, dtype=float) + + +def _nan_cumsum(values: np.ndarray, mask: np.ndarray, axis: int) -> np.ndarray: + """ + Running total along ``axis`` that counts NaN as zero. + + The result is NaN outside ``mask`` and where the total is zero. + """ + result = np.nancumsum(values, axis=axis) * mask result[result == 0] = np.nan return result -def _nan_diff(values: np.ndarray, axis: int) -> np.ndarray: - """First difference keeping the shape with NaN in the first slot, like pandas.""" - result = np.full_like(values, np.nan) - head = [slice(None)] * values.ndim - tail = [slice(None)] * values.ndim - head[axis] = slice(1, None) - tail[axis] = slice(None, -1) - result[tuple(head)] = values[tuple(head)] - values[tuple(tail)] +def _nan_diff( + values: np.ndarray, mask: np.ndarray, axis: int, periods: int = 1 +) -> np.ndarray: + """ + Pandas ``diff`` along ``axis`` that counts NaN as zero. + + The result is NaN where either cell is outside ``mask``, where there is no + cell to subtract and where the difference is zero. + """ + + def pandas_diff(array: np.ndarray) -> np.ndarray: + moved = np.moveaxis(array, axis, 0) + frame = pd.DataFrame(moved.reshape(moved.shape[0], -1)) + diff = frame.diff(periods=periods).to_numpy().reshape(moved.shape) + return np.moveaxis(diff, 0, axis).copy() + + result = pandas_diff(np.where(np.isnan(values), 0.0, values)) + result[np.isnan(pandas_diff(np.broadcast_to(mask, values.shape)))] = np.nan result[result == 0] = np.nan return result +SHAPE_PRESERVING_EXPECTED = {"cumsum": _nan_cumsum, "diff": _nan_diff} + + def _assert_same_labels(result: Triangle, triangle: Triangle) -> None: assert result.index.equals(triangle.index) assert result.columns.equals(triangle.columns) @@ -229,50 +261,68 @@ def _assert_same_labels(result: Triangle, triangle: Triangle) -> None: assert result.development.equals(triangle.development) -@pytest.mark.parametrize( - ("method", "expected_func"), [("cumsum", _nan_cumsum), ("diff", _nan_diff)] -) -@pytest.mark.parametrize("axis", ["index", "columns", "origin", "development"]) -def test_triangle_shape_preserving_values(clrd, method, expected_func, axis): +@pytest.mark.parametrize("method", ["cumsum", "diff"]) +@pytest.mark.parametrize("axis", AXES) +def test_triangle_shape_preserving_values( + clrd: Triangle, method: str, axis: str +) -> None: """ Apply ``cumsum`` and ``diff`` along each axis without collapsing it. - The result is a Triangle with the input's shape, labels and backend; empty - cells stay NaN and the input triangle is not modified. + The result is a Triangle with the input's shape, labels and backend. A NaN + inside the triangle counts as zero, cells outside the triangle stay NaN and + the input triangle is not modified. """ triangle = clrd.iloc[13:16, :2, -5:, :5] before = triangle.copy() - values = triangle.set_backend("numpy").values - assert np.isnan(values).any() - expected = expected_func(values, triangle._get_axis(axis)) + values = _numpy_values(triangle) + mask = _triangle_mask(triangle) + # The slice has both kinds of NaN: zeros inside the triangle, empty outside. + assert np.isnan(values[..., ~np.isnan(mask)]).any() + assert np.isnan(mask).any() + expected = SHAPE_PRESERVING_EXPECTED[method](values, mask, AXES.index(axis)) result = getattr(triangle, method)(axis=axis, auto_sparse=False) assert result.shape == triangle.shape assert result.array_backend == triangle.array_backend _assert_same_labels(result, triangle) - np.testing.assert_allclose( - result.set_backend("numpy").values, expected, equal_nan=True - ) + np.testing.assert_allclose(_numpy_values(result), expected, equal_nan=True) assert triangle.equals(before) -def test_cumsum_development_inverts_cum_to_incr(raa): - """Cumulating incremental amounts along development restores the triangle.""" - result = raa.cum_to_incr().cumsum(axis="development") +def test_cumsum_development_inverts_cum_to_incr(clrd: Triangle) -> None: + """ + Cumulating incremental amounts along development restores the triangle. + + ``clrd`` has zero increments, which are stored as NaN inside the triangle. + """ + incremental = clrd.cum_to_incr() + inside = ~np.isnan(_triangle_mask(clrd)) + assert np.isnan(_numpy_values(incremental)[..., inside]).any() + + result = incremental.cumsum(axis="development") - assert result.shape == raa.shape + assert result.shape == clrd.shape np.testing.assert_allclose( - result.set_backend("numpy").values, - raa.set_backend("numpy").values, + _numpy_values(result), + _numpy_values(clrd), equal_nan=True, ) -def test_diff_development_matches_cum_to_incr(raa): - """Differencing along development gives the increments after the first age.""" - result = raa.diff(axis="development").set_backend("numpy").values - incremental = raa.cum_to_incr().set_backend("numpy").values +def test_diff_development_matches_cum_to_incr(clrd: Triangle) -> None: + """ + Differencing along development gives the increments after the first age. + + ``clrd`` has zero cumulative amounts, which are stored as NaN inside the + triangle. The first age has no earlier cell to subtract and is NaN. + """ + inside = ~np.isnan(_triangle_mask(clrd)) + assert np.isnan(_numpy_values(clrd)[..., inside]).any() + + result = _numpy_values(clrd.diff(axis="development")) + incremental = _numpy_values(clrd.cum_to_incr()) assert np.isnan(result[..., 0]).all() np.testing.assert_allclose(result[..., 1:], incremental[..., 1:], equal_nan=True) @@ -280,7 +330,9 @@ def test_diff_development_matches_cum_to_incr(raa): @pytest.mark.parametrize("method", ["cumsum", "diff"]) @pytest.mark.parametrize("axis", range(4)) -def test_shape_preserving_default_axis_first_non_singleton(clrd, method, axis): +def test_shape_preserving_default_axis_first_non_singleton( + clrd: Triangle, method: str, axis: int +) -> None: """With ``axis=None``, use the first non-singleton axis, like the reductions.""" indexer = [slice(11, 13), slice(0, 2), slice(0, 5), slice(0, 5)] for preceding_axis in range(axis): @@ -294,7 +346,7 @@ def test_shape_preserving_default_axis_first_non_singleton(clrd, method, axis): @pytest.mark.parametrize("method", ["cumsum", "diff"]) -def test_shape_preserving_singleton(raa, method): +def test_shape_preserving_singleton(raa: Triangle, method: str) -> None: """ A (1, 1, 1, 1) Triangle stays a Triangle instead of becoming a scalar. @@ -306,15 +358,15 @@ def test_shape_preserving_singleton(raa, method): assert result.shape == triangle.shape assert result.array_backend == triangle.array_backend _assert_same_labels(result, triangle) - values = result.set_backend("numpy").values + values = _numpy_values(result) if method == "diff": assert np.isnan(values).all() else: - np.testing.assert_allclose(values, triangle.set_backend("numpy").values) + np.testing.assert_allclose(values, _numpy_values(triangle)) @pytest.mark.parametrize("method", ["cumsum", "diff"]) -def test_shape_preserving_negative_axis(raa, method): +def test_shape_preserving_negative_axis(raa: Triangle, method: str) -> None: """Negative axis numbers count back from the development axis.""" result = getattr(raa, method)(axis=-1, auto_sparse=False) expected = getattr(raa, method)(axis="development", auto_sparse=False) @@ -322,7 +374,7 @@ def test_shape_preserving_negative_axis(raa, method): @pytest.mark.parametrize("method", ["cumsum", "diff"]) -def test_shape_preserving_empty_triangle(raa, method): +def test_shape_preserving_empty_triangle(raa: Triangle, method: str) -> None: """A Triangle with no index rows is returned empty, on the same backend.""" triangle = raa.iloc[:0, :, :, :] @@ -333,7 +385,7 @@ def test_shape_preserving_empty_triangle(raa, method): @pytest.mark.parametrize("method", ["cumsum", "diff"]) -def test_shape_preserving_auto_sparse(raa, method): +def test_shape_preserving_auto_sparse(raa: Triangle, method: str) -> None: """ By default a small sparse result is converted to numpy, like the reductions. @@ -349,64 +401,85 @@ def test_shape_preserving_auto_sparse(raa, method): assert default.equals(kept) -@pytest.mark.parametrize("skipna", [True, False]) -def test_cumsum_internal_missing_cells(raa, skipna): +def test_cumsum_counts_nan_inside_triangle_as_zero(raa: Triangle) -> None: """ - Continue the total after a missing cell only with ``skipna=True``. + Carry the running total through a NaN inside the triangle. - The missing cell stays missing either way, and a zero total is stored as NaN. + Triangles store zeros as NaN, so that cell is a zero and not a gap. A zero + total is stored as NaN in turn, and the input triangle is not modified. """ triangle = raa.iloc[:, :, :1, :5] triangle.values = triangle.get_array_module().array( np.array([2.0, np.nan, 3.0, -5.0, 4.0]).reshape(triangle.shape) ) before = triangle.copy() - result = triangle.cumsum(axis="development", skipna=skipna, auto_sparse=False) - expected = [2.0, np.nan, 5.0, np.nan, 4.0] if skipna else [2.0] + [np.nan] * 4 + + result = triangle.cumsum(axis="development", auto_sparse=False) np.testing.assert_allclose( - result.set_backend("numpy").values.ravel(), expected, equal_nan=True + _numpy_values(result).ravel(), + [2.0, 2.0, 5.0, np.nan, 4.0], + equal_nan=True, ) assert triangle.equals(before) +def test_diff_counts_nan_inside_triangle_as_zero(raa: Triangle) -> None: + """ + Subtract a NaN inside the triangle, and subtract from one, as a zero. + + A zero difference is stored as NaN in turn, and the first cell has no + earlier cell to subtract. + """ + triangle = raa.iloc[:, :, :1, :5] + triangle.values = triangle.get_array_module().array( + np.array([2.0, np.nan, 3.0, 3.0, 4.0]).reshape(triangle.shape) + ) + + result = triangle.diff(axis="development", auto_sparse=False) + + np.testing.assert_allclose( + _numpy_values(result).ravel(), + [np.nan, -2.0, 3.0, np.nan, 1.0], + equal_nan=True, + ) + + @pytest.mark.parametrize("axis", range(4)) @pytest.mark.parametrize("periods", [0, 2, -1, -2, 10, 12]) -def test_diff_periods(clrd, axis, periods): - """Match pandas ``diff`` for each axis and ``periods``, with zeros as NaN.""" +def test_diff_periods(clrd: Triangle, axis: int, periods: int) -> None: + """ + Match pandas ``diff`` for each axis and ``periods``. + + A NaN inside the triangle counts as zero, and a difference with a cell + outside the triangle is NaN. + """ triangle = clrd.iloc[13:16, :2, -5:, :5] - values = np.moveaxis(triangle.set_backend("numpy").values, axis, 0) - expected = ( - pd - .DataFrame(values.reshape(values.shape[0], -1)) - .diff(periods=periods) - .to_numpy() - .reshape(values.shape) - ) - expected = np.moveaxis(expected, 0, axis) - expected[expected == 0] = np.nan + values = _numpy_values(triangle) + expected = _nan_diff(values, _triangle_mask(triangle), axis, periods) result = triangle.diff(axis=axis, periods=periods, auto_sparse=False) assert result.shape == triangle.shape assert result.array_backend == triangle.array_backend _assert_same_labels(result, triangle) - np.testing.assert_allclose( - result.set_backend("numpy").values, expected, equal_nan=True - ) + np.testing.assert_allclose(_numpy_values(result), expected, equal_nan=True) -def test_diff_periods_must_be_integer(raa): +def test_diff_periods_must_be_integer(raa: Triangle) -> None: """A non-integer ``periods`` raises instead of being truncated.""" + not_an_integer: Any = 1.5 with pytest.raises(TypeError): - raa.diff(axis="development", periods=1.5) + raa.diff(axis="development", periods=not_an_integer) -def test_sparse_transforms_do_not_densify(raa, monkeypatch): +def test_sparse_transforms_do_not_densify( + raa: Triangle, monkeypatch: pytest.MonkeyPatch +) -> None: """``cumsum`` and ``diff`` on a sparse Triangle never call ``todense``.""" triangle = raa.set_backend("sparse") - def reject_densification(*_args, **_kwargs): + def reject_densification(*_args: object, **_kwargs: object) -> None: pytest.fail("Transform unexpectedly densified its sparse input") with monkeypatch.context() as patch: @@ -417,7 +490,7 @@ def reject_densification(*_args, **_kwargs): assert cumulative.array_backend == difference.array_backend == "sparse" -def test_groupby_cumsum(clrd): +def test_groupby_cumsum(clrd: Triangle) -> None: """ Cumulate within each group and keep every row in the original order. @@ -427,66 +500,65 @@ def test_groupby_cumsum(clrd): triangle = clrd.iloc[:12, :2, -4:, :4] before = triangle.copy() assert not triangle.index["LOB"].is_monotonic_increasing - values = triangle.set_backend("numpy").values + values = _numpy_values(triangle) + mask = _triangle_mask(triangle) expected = np.empty_like(values) for _, rows in triangle.index.groupby("LOB").indices.items(): - expected[rows] = _nan_cumsum(values[rows], axis=0) + expected[rows] = _nan_cumsum(values[rows], mask, axis=0) result = triangle.groupby("LOB").cumsum(auto_sparse=False) assert result.shape == triangle.shape assert result.array_backend == triangle.array_backend _assert_same_labels(result, triangle) - np.testing.assert_allclose( - result.set_backend("numpy").values, expected, equal_nan=True - ) + np.testing.assert_allclose(_numpy_values(result), expected, equal_nan=True) assert triangle.equals(before) @pytest.mark.parametrize("axis", [1, 2, 3]) -def test_groupby_cumsum_other_axes(clrd, axis): +def test_groupby_cumsum_other_axes(clrd: Triangle, axis: Literal[1, 2, 3]) -> None: """Group along the columns, origin and development axes as well.""" triangle = clrd.iloc[13:16, :4, -4:, :4] groups = ["a", "b", "a", "b"] - values = triangle.set_backend("numpy").values + values = _numpy_values(triangle) + mask = _triangle_mask(triangle) expected = np.empty_like(values) for positions in ([0, 2], [1, 3]): indexer = [slice(None)] * axis + [positions] - expected[tuple(indexer)] = _nan_cumsum(values[tuple(indexer)], axis) + # The group keeps only its own origins or developments of the mask. + group_mask = mask[tuple(indexer[2:])] + expected[tuple(indexer)] = _nan_cumsum(values[tuple(indexer)], group_mask, axis) result = triangle.groupby(groups, axis=axis).cumsum(auto_sparse=False) assert result.shape == triangle.shape assert result.array_backend == triangle.array_backend _assert_same_labels(result, triangle) - np.testing.assert_allclose( - result.set_backend("numpy").values, expected, equal_nan=True - ) + np.testing.assert_allclose(_numpy_values(result), expected, equal_nan=True) @pytest.mark.parametrize("all_missing", [True, False]) -def test_groupby_cumsum_missing_keys(clrd, all_missing): +def test_groupby_cumsum_missing_keys(clrd: Triangle, all_missing: bool) -> None: """Labels with a missing group key are NaN; the other groups still cumulate.""" triangle = clrd.iloc[:12, :2, -4:, :4] labels = triangle.index.copy() labels.loc[labels.index if all_missing else [0], "LOB"] = None triangle.index = labels - values = triangle.set_backend("numpy").values + values = _numpy_values(triangle) + mask = _triangle_mask(triangle) expected = np.full_like(values, np.nan) for rows in labels.groupby("LOB").indices.values(): - expected[rows] = _nan_cumsum(values[rows], axis=0) + expected[rows] = _nan_cumsum(values[rows], mask, axis=0) result = triangle.groupby("LOB").cumsum(auto_sparse=False) assert result.shape == triangle.shape assert result.array_backend == triangle.array_backend _assert_same_labels(result, triangle) - np.testing.assert_allclose( - result.set_backend("numpy").values, expected, equal_nan=True - ) + np.testing.assert_allclose(_numpy_values(result), expected, equal_nan=True) -def test_groupby_cumsum_auto_sparse(clrd): +def test_groupby_cumsum_auto_sparse(clrd: Triangle) -> None: """ By default a small sparse result is converted to numpy, like the reductions. @@ -500,21 +572,3 @@ def test_groupby_cumsum_auto_sparse(clrd): assert default.array_backend == "numpy" assert kept.array_backend == "sparse" assert default.equals(kept) - - -@pytest.mark.parametrize("skipna", [True, False]) -def test_groupby_cumsum_skipna(raa, skipna): - """``skipna`` applies within each group and does not carry across groups.""" - triangle = raa.iloc[:, :, :1, :4] - triangle.values = triangle.get_array_module().array( - np.array([2.0, np.nan, 3.0, 4.0]).reshape(triangle.shape) - ) - expected = [2.0, np.nan, 5.0, 4.0] if skipna else [2.0, np.nan, np.nan, 4.0] - - result = triangle.groupby(["a", "a", "a", "b"], axis=3).cumsum( - skipna=skipna, auto_sparse=False - ) - - np.testing.assert_allclose( - result.set_backend("numpy").values.ravel(), expected, equal_nan=True - )