From 3c1cfa4ecbb1f8746123d62b62b9807d210f676b Mon Sep 17 00:00:00 2001 From: Simon Koch-Sultan Date: Tue, 10 Nov 2026 10:15:40 -0500 Subject: [PATCH 1/3] Use cumulative amounts for Mack risk on incremental triangles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MackChainladder fit on an incremental triangle multiplied incremental amounts in the Mack recursion, so parameter_risk_, process_risk_, full_std_err_ and the standard errors built on them were wrong. fit, predict and full_std_err_ now use the cumulative form of an incremental triangle for these estimates, and summary_ takes the latest diagonal from the estimator, which sums an incremental triangle. X_ and full_triangle_ stay incremental. Closes #1571. Developed with an AI assistant (Árbol, running Muse). --- chainladder/methods/mack.py | 41 ++++++++++++++++++-------- chainladder/methods/tests/test_mack.py | 33 +++++++++++++++++++++ 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/chainladder/methods/mack.py b/chainladder/methods/mack.py index 96e4a4081..79d4cb919 100644 --- a/chainladder/methods/mack.py +++ b/chainladder/methods/mack.py @@ -157,17 +157,22 @@ def fit(self, X, y=None, sample_weight=None): 2012 7162.2 693.2 2013 14396.9 901.4 """ + from chainladder.core.common import _get_full_triangle + super().fit(X, y, sample_weight) if "sigma_" not in self.X_: raise ValueError("Triangle not compatible with MackChainladder") + # The Mack recursion multiplies cumulative amounts, so the risk + # estimates use the cumulative form of an incremental triangle. + X_cum = self.X_ if self.X_.is_cumulative else self.X_.incr_to_cum() # Caching full_triangle_ for fit as it is called a lot - self.X_._full_triangle_ = self.full_triangle_ - self.parameter_risk_ = self._mack_recursion("parameter_risk_", self.X_) - self.process_risk_ = self._mack_recursion("process_risk_", self.X_) + X_cum._full_triangle_ = _get_full_triangle(X_cum, self.ultimate_) + self.parameter_risk_ = self._mack_recursion("parameter_risk_", X_cum) + self.process_risk_ = self._mack_recursion("process_risk_", X_cum) self.total_parameter_risk_ = self._mack_recursion( - "total_parameter_risk_", self.X_ + "total_parameter_risk_", X_cum ) - del self.X_._full_triangle_ + del X_cum._full_triangle_ self.process_variance_ = self._include_process_variance() return self @@ -231,17 +236,20 @@ def predict(self, X, sample_weight=None): X_new.std_err_ = getattr(X_new, "std_err_", self.X_.std_err_) X_new.average_ = getattr(X_new, "average_", self.X_.average_) X_new.w_ = getattr(X_new, "w_", self.X_.w_) - X_new._full_triangle_ = X_new.full_triangle_ - X_new.parameter_risk_ = self._mack_recursion("parameter_risk_", X_new) - X_new.process_risk_ = self._mack_recursion("process_risk_", X_new) + # The Mack recursion multiplies cumulative amounts, so the risk + # estimates use the cumulative form of an incremental triangle. + X_cum = X_new if X_new.is_cumulative else X_new.incr_to_cum() + X_cum._full_triangle_ = X_cum.full_triangle_ + X_new.parameter_risk_ = self._mack_recursion("parameter_risk_", X_cum) + X_new.process_risk_ = self._mack_recursion("process_risk_", X_cum) X_new.total_process_risk_ = (X_new.process_risk_**2).sum(axis="origin").sqrt() X_new.total_parameter_risk_ = self._mack_recursion( - "total_parameter_risk_", X_new + "total_parameter_risk_", X_cum ) - X_new.full_std_err_ = self._get_full_std_err_(X_new) + X_new.full_std_err_ = self._get_full_std_err_(X_cum) X_new.total_mack_std_err_ = self._get_total_mack_std_err_(X_new) X_new.mack_std_err_ = (X_new.parameter_risk_**2 + X_new.process_risk_**2).sqrt() - del X_new._full_triangle_ + del X_cum._full_triangle_ return X_new @property @@ -281,7 +289,14 @@ def full_std_err_(self): 2012 0.039675 0.034017 0.026776 0.008976 0.001243 0.001324 0.0 2013 0.035752 0.030671 0.024143 0.008094 0.001121 0.001193 0.0 """ - return self._get_full_std_err_(self.X_) + X = self.X_ + if not X.is_cumulative: + # The weights use cumulative amounts. + from chainladder.core.common import _get_full_triangle + + X = X.incr_to_cum() + X._full_triangle_ = _get_full_triangle(X, self.ultimate_) + return self._get_full_std_err_(X) def _get_full_std_err_(self, X=None): from chainladder.utils.utility_functions import num_to_nan @@ -506,7 +521,7 @@ def summary_(self): backend = obj.array_backend # forcing these four triangles to the same backend cols = ( - self.X_.latest_diagonal.set_backend(backend).values, + self.latest_diagonal.set_backend(backend).values, self.ibnr_.set_backend(backend).values, self.ultimate_.set_backend(backend).values, self.mack_std_err_.set_backend(backend).values[..., -1:], diff --git a/chainladder/methods/tests/test_mack.py b/chainladder/methods/tests/test_mack.py index 9e9bfb9a0..f48b666fa 100644 --- a/chainladder/methods/tests/test_mack.py +++ b/chainladder/methods/tests/test_mack.py @@ -65,6 +65,39 @@ def test_mack1999_hardcode(): assert np.allclose(ibnr_se, np.nan_to_num(ibnr_rhs, nan=0), atol=1, equal_nan=True) +def test_mack_incremental_matches_cumulative(): + """An incremental triangle gives the same Mack estimates as its cumulative form.""" + attrs = [ + "parameter_risk_", + "process_risk_", + "total_parameter_risk_", + "total_process_risk_", + "mack_std_err_", + "full_std_err_", + ] + for sample in ["raa", "mw2008", "genins"]: + cum = cl.load_sample(sample) + incr = cum.cum_to_incr() + model_cum = cl.MackChainladder().fit(cum) + model_incr = cl.MackChainladder().fit(incr) + assert not model_incr.X_.is_cumulative + pairs = [ + (model_cum, model_incr), + (model_cum.predict(cum), model_cum.predict(incr)), + ] + for a, b in pairs: + for attr in attrs: + assert np.allclose( + getattr(a, attr).values, getattr(b, attr).values, equal_nan=True + ), (sample, attr) + assert np.allclose( + a.total_mack_std_err_.values, b.total_mack_std_err_.values + ) + assert np.allclose( + model_cum.summary_.values, model_incr.summary_.values, equal_nan=True + ) + + def test_mack1994_hardcode(raa): """ Reconciles key MackChainladder statistics to values provided in the paper From 88b3dfbd8732b2ffb84e64c383f6d2c5d0ddee2f Mon Sep 17 00:00:00 2001 From: Simon Koch-Sultan Date: Wed, 11 Nov 2026 09:21:30 +0000 Subject: [PATCH 2/3] Copy X_ unconditionally for the Mack risk estimates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fit, predict and full_std_err_ now always take incr_to_cum(), which returns a copy and leaves a cumulative triangle's values unchanged, so the cached full triangle stays on the copy and the del lines are gone. _get_full_std_err_ builds its full_triangle_ fallback only when X has no cached full triangle; getattr evaluated it on every call, which would have doubled full_std_err_'s work on cumulative input. Outputs are bit-identical. The incremental test runs on the raa, mw2008 and genins fixtures, on the numpy and sparse backends. conftest.py gains main's mw2008 and mw2014 entries unchanged, so the branch still merges cleanly with main. Developed with an AI assistant (Árbol, running Muse). --- chainladder/methods/mack.py | 25 ++++++------ chainladder/methods/tests/test_mack.py | 54 ++++++++++++++++---------- conftest.py | 8 ++++ 3 files changed, 53 insertions(+), 34 deletions(-) diff --git a/chainladder/methods/mack.py b/chainladder/methods/mack.py index 79d4cb919..a99d9f596 100644 --- a/chainladder/methods/mack.py +++ b/chainladder/methods/mack.py @@ -163,8 +163,8 @@ def fit(self, X, y=None, sample_weight=None): if "sigma_" not in self.X_: raise ValueError("Triangle not compatible with MackChainladder") # The Mack recursion multiplies cumulative amounts, so the risk - # estimates use the cumulative form of an incremental triangle. - X_cum = self.X_ if self.X_.is_cumulative else self.X_.incr_to_cum() + # estimates use a cumulative copy of X_. + X_cum = self.X_.incr_to_cum() # Caching full_triangle_ for fit as it is called a lot X_cum._full_triangle_ = _get_full_triangle(X_cum, self.ultimate_) self.parameter_risk_ = self._mack_recursion("parameter_risk_", X_cum) @@ -172,7 +172,6 @@ def fit(self, X, y=None, sample_weight=None): self.total_parameter_risk_ = self._mack_recursion( "total_parameter_risk_", X_cum ) - del X_cum._full_triangle_ self.process_variance_ = self._include_process_variance() return self @@ -237,8 +236,8 @@ def predict(self, X, sample_weight=None): X_new.average_ = getattr(X_new, "average_", self.X_.average_) X_new.w_ = getattr(X_new, "w_", self.X_.w_) # The Mack recursion multiplies cumulative amounts, so the risk - # estimates use the cumulative form of an incremental triangle. - X_cum = X_new if X_new.is_cumulative else X_new.incr_to_cum() + # estimates use a cumulative copy of X_new. + X_cum = X_new.incr_to_cum() X_cum._full_triangle_ = X_cum.full_triangle_ X_new.parameter_risk_ = self._mack_recursion("parameter_risk_", X_cum) X_new.process_risk_ = self._mack_recursion("process_risk_", X_cum) @@ -249,7 +248,6 @@ def predict(self, X, sample_weight=None): X_new.full_std_err_ = self._get_full_std_err_(X_cum) X_new.total_mack_std_err_ = self._get_total_mack_std_err_(X_new) X_new.mack_std_err_ = (X_new.parameter_risk_**2 + X_new.process_risk_**2).sqrt() - del X_cum._full_triangle_ return X_new @property @@ -289,13 +287,11 @@ def full_std_err_(self): 2012 0.039675 0.034017 0.026776 0.008976 0.001243 0.001324 0.0 2013 0.035752 0.030671 0.024143 0.008094 0.001121 0.001193 0.0 """ - X = self.X_ - if not X.is_cumulative: - # The weights use cumulative amounts. - from chainladder.core.common import _get_full_triangle + from chainladder.core.common import _get_full_triangle - X = X.incr_to_cum() - X._full_triangle_ = _get_full_triangle(X, self.ultimate_) + # The weights use cumulative amounts. + X = self.X_.incr_to_cum() + X._full_triangle_ = _get_full_triangle(X, self.ultimate_) return self._get_full_std_err_(X) def _get_full_std_err_(self, X=None): @@ -304,7 +300,10 @@ def _get_full_std_err_(self, X=None): obj = X.copy() xp = obj.get_array_module() lxp = X.ldf_.get_array_module() - full = getattr(X, "_full_triangle_", self.full_triangle_) + # The fallback is built only when X carries no cached full triangle. + full = getattr(X, "_full_triangle_", None) + if full is None: + full = self.full_triangle_ avg = {"regression": 0, "volume": 1, "simple": 2} avg = [avg.get(item, item) for item in X.average_] val = xp.broadcast_to(xp.array(avg + [avg[-1]]), X.shape) diff --git a/chainladder/methods/tests/test_mack.py b/chainladder/methods/tests/test_mack.py index f48b666fa..35f2fd89c 100644 --- a/chainladder/methods/tests/test_mack.py +++ b/chainladder/methods/tests/test_mack.py @@ -65,7 +65,7 @@ def test_mack1999_hardcode(): assert np.allclose(ibnr_se, np.nan_to_num(ibnr_rhs, nan=0), atol=1, equal_nan=True) -def test_mack_incremental_matches_cumulative(): +def _assert_incremental_matches_cumulative(cum): """An incremental triangle gives the same Mack estimates as its cumulative form.""" attrs = [ "parameter_risk_", @@ -75,27 +75,39 @@ def test_mack_incremental_matches_cumulative(): "mack_std_err_", "full_std_err_", ] - for sample in ["raa", "mw2008", "genins"]: - cum = cl.load_sample(sample) - incr = cum.cum_to_incr() - model_cum = cl.MackChainladder().fit(cum) - model_incr = cl.MackChainladder().fit(incr) - assert not model_incr.X_.is_cumulative - pairs = [ - (model_cum, model_incr), - (model_cum.predict(cum), model_cum.predict(incr)), - ] - for a, b in pairs: - for attr in attrs: - assert np.allclose( - getattr(a, attr).values, getattr(b, attr).values, equal_nan=True - ), (sample, attr) + incr = cum.cum_to_incr() + model_cum = cl.MackChainladder().fit(cum) + model_incr = cl.MackChainladder().fit(incr) + assert not model_incr.X_.is_cumulative + pairs = [ + (model_cum, model_incr), + (model_cum.predict(cum), model_cum.predict(incr)), + ] + for a, b in pairs: + for attr in attrs: assert np.allclose( - a.total_mack_std_err_.values, b.total_mack_std_err_.values - ) - assert np.allclose( - model_cum.summary_.values, model_incr.summary_.values, equal_nan=True - ) + getattr(a, attr).set_backend("numpy").values, + getattr(b, attr).set_backend("numpy").values, + equal_nan=True, + ), attr + assert np.allclose(a.total_mack_std_err_.values, b.total_mack_std_err_.values) + assert np.allclose( + model_cum.summary_.set_backend("numpy").values, + model_incr.summary_.set_backend("numpy").values, + equal_nan=True, + ) + + +def test_mack_incremental_matches_cumulative_raa(raa): + _assert_incremental_matches_cumulative(raa) + + +def test_mack_incremental_matches_cumulative_mw2008(mw2008): + _assert_incremental_matches_cumulative(mw2008) + + +def test_mack_incremental_matches_cumulative_genins(genins): + _assert_incremental_matches_cumulative(genins) def test_mack1994_hardcode(raa): diff --git a/conftest.py b/conftest.py index 7818dadaf..9ad2c7a87 100644 --- a/conftest.py +++ b/conftest.py @@ -49,6 +49,14 @@ "data": "xyz", "runs": ["normal_run", "sparse_only_run"], }, + "mw2008": { + "data": "mw2008", + "runs": ["normal_run", "sparse_only_run"], + }, + "mw2014": { + "data": "mw2014", + "runs": ["normal_run", "sparse_only_run"], + }, } From a95cd192daa66829b977e894335713db59cb9e0c Mon Sep 17 00:00:00 2001 From: Simon Koch-Sultan Date: Wed, 11 Nov 2026 07:14:01 -0500 Subject: [PATCH 3/3] Read the cached full triangle directly in _get_full_std_err_ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fit, predict and full_std_err_ each cache the full triangle on X before calling _get_full_std_err_, so its fallback to self.full_triangle_ never ran and Codecov counted it as uncovered. Developed with an AI assistant (Árbol, running Muse). --- chainladder/methods/mack.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/chainladder/methods/mack.py b/chainladder/methods/mack.py index a99d9f596..925d6d5f7 100644 --- a/chainladder/methods/mack.py +++ b/chainladder/methods/mack.py @@ -300,10 +300,8 @@ def _get_full_std_err_(self, X=None): obj = X.copy() xp = obj.get_array_module() lxp = X.ldf_.get_array_module() - # The fallback is built only when X carries no cached full triangle. - full = getattr(X, "_full_triangle_", None) - if full is None: - full = self.full_triangle_ + # fit, predict and full_std_err_ cache the full triangle on X first. + full = X._full_triangle_ avg = {"regression": 0, "volume": 1, "simple": 2} avg = [avg.get(item, item) for item in X.average_] val = xp.broadcast_to(xp.array(avg + [avg[-1]]), X.shape)