Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 72 additions & 0 deletions chainladder/methods/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,78 @@ def _align_cdf(self, X, sample_weight=None):
"""Vertically align CDF to origin period latest diagonal."""
return X.cdf_.align_pattern(X, sample_weight)

@staticmethod
def _slice_weight_origins(X, sample_weight):
"""
Slice ``sample_weight`` to the origin range it shares with ``X``.

The methods read ``sample_weight`` by position along the origin axis.
When ``X`` and ``sample_weight`` each have more than one origin, this
keeps the weight from the later of the two first origins to the
earlier of the two last origins, and gives NaN weight to origins of
``X`` outside that range. A warning names those origins of ``X``.

Both triangles need consecutive origins at the same origin grain.
A ValueError is raised when the grains differ, when the ranges share
no origin, or when the origins in the shared range do not line up.

Returns the weight and the origins of ``X`` that it lacks, which get
no ultimate.
"""
none = pd.DatetimeIndex([])
if sample_weight is None or len(X.odims) == 1 or len(sample_weight.odims) == 1:
return sample_weight, none
if X.origin_grain != sample_weight.origin_grain:
raise ValueError(
f"X has origin grain {X.origin_grain} but sample_weight has "
f"origin grain {sample_weight.origin_grain}. They must match."
)
x_origins = pd.DatetimeIndex(X.odims)
w_origins = pd.DatetimeIndex(sample_weight.odims)
if x_origins.equals(w_origins):
return sample_weight, none
start = max(x_origins[0], w_origins[0])
end = min(x_origins[-1], w_origins[-1])
x_shared = (x_origins >= start) & (x_origins <= end)
w_shared = (w_origins >= start) & (w_origins <= end)
if not x_shared.any():
raise ValueError("X and sample_weight share no origin.")
if not x_origins[x_shared].equals(w_origins[w_shared]):
raise ValueError(
"The origins of X and sample_weight do not line up in the range "
"they share. Both need consecutive origins."
)
if not x_shared.all():
import warnings

warnings.warn(
"sample_weight is sliced to the origins it shares with X. "
"Origins of X with no weight get no ultimate: "
f"{X.origin[~x_shared].astype(str).tolist()}."
)
backend = sample_weight.array_backend
obj = sample_weight.set_backend("numpy")
xp = obj.get_array_module()
values = xp.full(
obj.shape[:2] + (len(x_origins), obj.shape[3]), xp.nan, dtype=float
)
values[:, :, x_shared, :] = obj.values[:, :, w_shared, :]
obj.values = values
obj.odims = X.odims.copy()
return obj.set_backend(backend), x_origins[~x_shared]

@staticmethod
def _drop_ultimates(ultimate, origins):
"""Set ``ultimate`` to NaN at ``origins``, which have no weight."""
if len(origins) == 0:
return ultimate
backend = ultimate.array_backend
obj = ultimate.set_backend("numpy")
values = obj.values.copy()
values[:, :, pd.DatetimeIndex(obj.odims).isin(origins), :] = float("nan")
obj.values = values
return obj.set_backend(backend)

def _set_ult_attr(self, ultimate):
"""Ultimate scaffolding"""
from chainladder import options
Expand Down
4 changes: 4 additions & 0 deletions chainladder/methods/benktander.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,11 +194,13 @@ def fit(self, X, y=None, sample_weight=None):
2008 50708.755370

"""
sample_weight, missing = self._slice_weight_origins(X, sample_weight)
if sample_weight is None:
raise ValueError("sample_weight is required.")
super().fit(X, y, sample_weight)
self.expectation_ = self._get_benktander_aprioris(X, sample_weight)
self.ultimate_ = self._get_ultimate(self.X_, self.expectation_)
self.ultimate_ = self._drop_ultimates(self.ultimate_, missing)
self.process_variance_ = self._include_process_variance()
return self

Expand Down Expand Up @@ -250,11 +252,13 @@ def predict(self, X, sample_weight=None):
2012 15914.716737
2013 17193.715555
"""
sample_weight, missing = self._slice_weight_origins(X, sample_weight)
if sample_weight is None:
raise ValueError("sample_weight is required.")
X_new = super().predict(X, sample_weight)
X_new.expectation_ = self._get_benktander_aprioris(X, sample_weight)
X_new.ultimate_ = self._get_ultimate(X_new, X_new.expectation_)
X_new.ultimate_ = self._drop_ultimates(X_new.ultimate_, missing)
X_new.n_iters = self.n_iters
X_new.apriori = self.apriori
return X_new
Expand Down
6 changes: 6 additions & 0 deletions chainladder/methods/capecod.py
Original file line number Diff line number Diff line change
Expand Up @@ -237,12 +237,16 @@ def fit(self, X, y=None, sample_weight=None):
raise ValueError("sample_weight is required.")
self.apriori = 1.0
self.X_ = self.validate_X(X)
sample_weight, missing = self._slice_weight_origins(X, sample_weight)
sample_weight = sample_weight.set_backend(self.X_.array_backend)
self.apriori_, self.detrended_apriori_ = self._get_capecod_aprioris(
self.X_, sample_weight
)
self.expectation_ = sample_weight * self.detrended_apriori_
super().fit(X, y, self.expectation_)
if len(missing):
self.ultimate_ = self._drop_ultimates(self.ultimate_, missing)
self.process_variance_ = self._include_process_variance()
return self

def _get_capecod_aprioris(self, X, sample_weight):
Expand Down Expand Up @@ -333,6 +337,7 @@ def predict(self, X, sample_weight=None):
2012 15991.144199
2013 17489.630279
"""
sample_weight, missing = self._slice_weight_origins(X, sample_weight)
if sample_weight is None:
raise ValueError("sample_weight is required.")
X_new = X.copy()
Expand All @@ -350,6 +355,7 @@ def predict(self, X, sample_weight=None):
)
X_new.expectation_ = sample_weight * detrended_apriori_
X_new = super().predict(X_new, X_new.expectation_)
X_new.ultimate_ = self._drop_ultimates(X_new.ultimate_, missing)
X_new.apriori_ = apriori_
X_new.detrended_apriori_ = detrended_apriori_
return X_new
Expand Down
113 changes: 113 additions & 0 deletions chainladder/methods/tests/test_sample_weight.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
"""Tests for slicing ``sample_weight`` to the origins it shares with ``X`` (#1646)."""

import numpy as np
import pandas as pd
import pytest

import chainladder as cl

raa = cl.load_sample("RAA")
years = list(range(1981, 1991))


def _premium(origins, values=None):
if values is None:
values = [20000.0 + 2000.0 * (o - 1982) for o in origins]
df = pd.DataFrame({"origin": [str(o) for o in origins], "premium": values})
return cl.Triangle(df, origin="origin", columns="premium", cumulative=True)


def _same_labels(origins):
"""The premium of ``_premium`` on raa's origins, NaN outside ``origins``."""
values = [20000.0 + 2000.0 * (y - 1982) if y in origins else np.nan for y in years]
return _premium(years, values)


@pytest.mark.parametrize(
"estimator",
[cl.BornhuetterFerguson, cl.Benktander, cl.ExpectedLoss, cl.CapeCod],
)
def test_shifted_sample_weight_is_sliced_to_the_shared_origins(estimator):
"""A weight labelled 1982-1991 on raa (1981-1990) uses 1982-1990."""
shifted = _premium(range(1982, 1992))
matched = _same_labels(range(1982, 1991))
expected = estimator().fit(raa, sample_weight=matched).ultimate_
expected = expected.values[0, 0, 1:, 0]
with pytest.warns(UserWarning, match=r"no ultimate: \['1981'\]"):
est = estimator().fit(raa, sample_weight=shifted)
with pytest.warns(UserWarning, match="sliced to the origins it shares"):
predicted = est.predict(raa, sample_weight=shifted)
for ultimate in (est.ultimate_, predicted.ultimate_):
assert np.isnan(ultimate.values[0, 0, 0, 0])
np.testing.assert_allclose(ultimate.values[0, 0, 1:, 0], expected)


def test_bornhuetter_ferguson_uses_the_premium_labelled_with_the_origin():
"""The 1990 ultimate uses the premium labelled 1990, from #1646."""
shifted = _premium(range(1982, 1992))
with pytest.warns(UserWarning):
est = cl.BornhuetterFerguson(apriori=0.6).fit(raa, sample_weight=shifted)
assert est.ultimate_.values[0, 0, -1, 0] == pytest.approx(21241.54, abs=0.01)


def test_longer_sample_weight_is_sliced_to_the_origins_of_x(recwarn):
"""A weight labelled 1979-1992 gives the same ultimates as one on 1981-1990."""
longer = cl.BornhuetterFerguson(apriori=0.6).fit(
raa, sample_weight=_premium(range(1979, 1993))
)
same = cl.BornhuetterFerguson(apriori=0.6).fit(raa, sample_weight=_premium(years))
np.testing.assert_allclose(longer.ultimate_.values, same.ultimate_.values)
assert not [w for w in recwarn if "sliced" in str(w.message)]


def test_shorter_sample_weight_leaves_origins_without_weight_with_no_ultimate():
"""A weight labelled 1983-1988 gives no ultimate for 1981, 1982, 1989, 1990."""
expected = cl.BornhuetterFerguson(apriori=0.6).fit(
raa, sample_weight=_same_labels(range(1983, 1989))
)
with pytest.warns(UserWarning, match=r"\['1981', '1982', '1989', '1990'\]"):
est = cl.BornhuetterFerguson(apriori=0.6).fit(
raa, sample_weight=_premium(range(1983, 1989))
)
ultimate = est.ultimate_.values[0, 0, :, 0]
assert np.isnan(ultimate[[0, 1, 8, 9]]).all()
np.testing.assert_allclose(ultimate[2:8], expected.ultimate_.values[0, 0, 2:8, 0])


def test_sample_weight_with_another_origin_grain_raises():
"""A quarterly weight on annual raa raises instead of lining up by position."""
origins = pd.date_range("1981-01-01", periods=40, freq="QS")
df = pd.DataFrame({"origin": origins, "premium": 5000.0})
quarterly = cl.Triangle(df, origin="origin", columns="premium", cumulative=True)
with pytest.raises(ValueError, match="origin grain"):
cl.BornhuetterFerguson(apriori=0.6).fit(raa, sample_weight=quarterly)


def test_origins_that_do_not_line_up_raise():
"""Odd-year X with an even-year weight raises, as with #1649's slices of raa."""
odd = raa[raa.origin.year.isin([1981, 1983, 1985, 1987, 1989])]
even = _premium(years)
even = even[even.origin.year.isin([1982, 1984, 1986, 1988, 1990])]
with pytest.raises(ValueError, match="do not line up"):
cl.BornhuetterFerguson(apriori=0.6).fit(odd, sample_weight=even)


def test_sample_weight_slicing_works_on_the_sparse_backend():
"""Slicing gives the same ultimates on the sparse backend."""
shifted = _premium(range(1982, 1992))
with pytest.warns(UserWarning):
dense = cl.BornhuetterFerguson(apriori=0.6).fit(raa, sample_weight=shifted)
with pytest.warns(UserWarning):
sparse = cl.BornhuetterFerguson(apriori=0.6).fit(
raa.set_backend("sparse"), sample_weight=shifted.set_backend("sparse")
)
np.testing.assert_allclose(
sparse.ultimate_.set_backend("numpy").values, dense.ultimate_.values
)


def test_single_origin_sample_weight_broadcasts():
"""A weight with one origin still applies to every origin of X."""
est = cl.BornhuetterFerguson(apriori=0.6).fit(raa, sample_weight=_premium([1990]))
assert est.ultimate_.shape == (1, 1, 10, 1)
assert not np.isnan(est.ultimate_.values).any()
Loading