From 4b5bca9f7d3d6efb25fa001e456d8a3c829be020 Mon Sep 17 00:00:00 2001 From: Simon Koch-Sultan Date: Fri, 13 Nov 2026 19:59:43 +0000 Subject: [PATCH 1/4] [FIX] Leave zero sigmas out of the log-linear sigma fill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit loglinear_interpolation replaced a zero sigma with 1e-320 and fitted its log (-736.8) at full weight, so a single zero-variation period could push the extrapolated sigmas to 1.8e308. Only positive sigmas now enter the fit, as in R's ChainLadder estimate.sigma, and zero sigmas are not filled. Where fewer than three positive sigmas remain, the fill falls back to Mack's approximation, which R's Mack.S.E also does for one point and for two points (undefined slope p-value). Closes #1609. Developed with an AI assistant (Árbol, running Muse). --- .../development/tests/test_development.py | 19 ++++++++++++++ chainladder/utils/weighted_regression.py | 25 +++++++++++++++---- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/chainladder/development/tests/test_development.py b/chainladder/development/tests/test_development.py index c6e8b78b9..605697ae7 100644 --- a/chainladder/development/tests/test_development.py +++ b/chainladder/development/tests/test_development.py @@ -1132,3 +1132,22 @@ def test_valid_params_pass_validation(raa): cl.Development(average=lambda age: "geometric", n_periods=[-1] * 9).fit(raa) dev = cl.Development(n_periods=4, average="simple") assert cl.read_json(dev.to_json()).get_params() == dev.get_params() + + +def test_loglinear_sigma_skips_zero_sigmas(clrd): + # GH 1609: zero sigmas used to enter the log-linear fit as log(1e-320). + tri = clrd["CumPaidLoss"] + + # One positive sigma: the fill falls back to Mack's approximation. + brotherhood = cl.Development().fit(tri.loc["Brotherhood Mut Ins Co", "prodliab"]) + assert np.nanmax(brotherhood.sigma_.set_backend("numpy").values) < 1e3 + assert np.isfinite(brotherhood.std_err_.set_backend("numpy").values).all() + + # Five positive sigmas (24-36 to 72-84) and two zeros (84-96, 96-108): + # the 12-24 and 108-120 fills come from the five positive sigmas only. + buckeye = cl.Development().fit(tri.loc["Buckeye Ins Grp", "othliab"]) + sigma = buckeye.sigma_.set_backend("numpy").values.ravel() + period = np.arange(1, sigma.size + 1) + slope, intercept = np.polyfit(period[1:6], np.log(sigma[1:6]), 1) + assert np.allclose(sigma[[0, 8]], np.exp(intercept + slope * period[[0, 8]])) + assert np.isclose(sigma[0], 7.1819, atol=1e-4) diff --git a/chainladder/utils/weighted_regression.py b/chainladder/utils/weighted_regression.py index 5e0a51a4b..6fd9959f5 100644 --- a/chainladder/utils/weighted_regression.py +++ b/chainladder/utils/weighted_regression.py @@ -262,17 +262,32 @@ def std_err_fill(self): return self def loglinear_interpolation(self, y): - """Use Cases: generally for filling in last element of sigma_""" + """ + Fill undefined sigmas by regressing log(sigma) on development period. + + As in R's ChainLadder (``estimate.sigma``), only positive sigmas enter + the regression: a zero sigma has no logarithm and would dominate the + fit. Zero sigmas are not filled. Where fewer than three positive sigmas + remain, the fill falls back to Mack's approximation. R's ``Mack.S.E`` + does the same for a single point, and for two points, where the + slope's p-value is undefined. + """ from chainladder.utils.utility_functions import num_to_nan xp = self.xp - ly = y.copy() - ly = xp.log(xp.where(ly == 0, 1e-320, ly)) + ly = xp.log(xp.where(y > 0, y, xp.nan)) w = xp.nan_to_num(ly * 0 + 1) + estimated = xp.nan_to_num(y * 0 + 1) reg = WeightedRegression(self.axis, False, xp=xp).fit(None, ly, w) slope, intercept = reg.slope_, reg.intercept_ - fill_ = xp.exp(reg.x * slope + intercept) * (1 - w) - out = xp.nan_to_num(y) + xp.nan_to_num(fill_) + fill_ = xp.nan_to_num(xp.exp(reg.x * slope + intercept) * (1 - estimated)) + use_mack = xp.sum(w, axis=self.axis, keepdims=True) < 3 + if use_mack.any(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=RuntimeWarning) + mack_fill = xp.nan_to_num(self.mack_interpolation(y)) * (1 - estimated) + fill_ = xp.where(use_mack, mack_fill, fill_) + out = xp.nan_to_num(y) + fill_ return num_to_nan(out) def mack_interpolation(self, y): From eb713311cad25108bd14702f142459b9aa27edb8 Mon Sep 17 00:00:00 2001 From: Simon Koch-Sultan Date: Mon, 16 Nov 2026 00:15:39 +0000 Subject: [PATCH 2/4] [FIX] Explain the sigma fill without R and test the Mack fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give the log-linear fill's reasons in the docstring: a zero sigma has no logarithm, and a line through two points leaves no residual to judge it by. Scope the fallback's error suppression to numpy's divide and invalid flags, which mack_interpolation sets where an earlier sigma is zero. Add a case where the fallback fills a sigma and compare it with sigma_interpolation="mack", and correct the comment on the case where Mack has nothing to fill from. Developed with an AI assistant (Árbol, running Muse). --- .../development/tests/test_development.py | 14 ++++++++++++- chainladder/utils/weighted_regression.py | 21 ++++++++++++------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/chainladder/development/tests/test_development.py b/chainladder/development/tests/test_development.py index 605697ae7..fe9fddc04 100644 --- a/chainladder/development/tests/test_development.py +++ b/chainladder/development/tests/test_development.py @@ -1138,11 +1138,23 @@ def test_loglinear_sigma_skips_zero_sigmas(clrd): # GH 1609: zero sigmas used to enter the log-linear fit as log(1e-320). tri = clrd["CumPaidLoss"] - # One positive sigma: the fill falls back to Mack's approximation. + # One positive sigma (36-48) and a zero (48-60): no line is fitted, and + # Mack's approximation has no two positive sigmas in a row, so the + # 12-24 and 24-36 sigmas stay NaN instead of 1.8e308. brotherhood = cl.Development().fit(tri.loc["Brotherhood Mut Ins Co", "prodliab"]) assert np.nanmax(brotherhood.sigma_.set_backend("numpy").values) < 1e3 assert np.isfinite(brotherhood.std_err_.set_backend("numpy").values).all() + # Two positive sigmas (12-24, 24-36), then none at 36-48 and zeros after: + # the 36-48 fill comes from Mack's approximation, the same value as + # under sigma_interpolation="mack". + great_amer = tri.loc["Great Amer Grp", "medmal"] + loglinear = cl.Development().fit(great_amer).sigma_.set_backend("numpy") + mack = cl.Development(sigma_interpolation="mack").fit(great_amer) + fill = loglinear.values.ravel()[2] + assert np.isclose(fill, mack.sigma_.set_backend("numpy").values.ravel()[2]) + assert np.isclose(fill, 14.9781, atol=1e-4) + # Five positive sigmas (24-36 to 72-84) and two zeros (84-96, 96-108): # the 12-24 and 108-120 fills come from the five positive sigmas only. buckeye = cl.Development().fit(tri.loc["Buckeye Ins Grp", "othliab"]) diff --git a/chainladder/utils/weighted_regression.py b/chainladder/utils/weighted_regression.py index 6fd9959f5..c2992d7b3 100644 --- a/chainladder/utils/weighted_regression.py +++ b/chainladder/utils/weighted_regression.py @@ -265,12 +265,15 @@ def loglinear_interpolation(self, y): """ Fill undefined sigmas by regressing log(sigma) on development period. - As in R's ChainLadder (``estimate.sigma``), only positive sigmas enter - the regression: a zero sigma has no logarithm and would dominate the - fit. Zero sigmas are not filled. Where fewer than three positive sigmas - remain, the fill falls back to Mack's approximation. R's ``Mack.S.E`` - does the same for a single point, and for two points, where the - slope's p-value is undefined. + Only positive sigmas enter the regression. A zero sigma has no + logarithm, and a stand-in such as 1e-320 puts a point near -737 on + the log scale that outweighs every other point in the least-squares + fit. Zero sigmas are left as they are. + + With fewer than three positive sigmas, the fill comes from + ``mack_interpolation`` instead. One point gives no slope, and a line + through two points fits both exactly, so no residual is left to judge + it by. """ from chainladder.utils.utility_functions import num_to_nan @@ -283,8 +286,10 @@ def loglinear_interpolation(self, y): fill_ = xp.nan_to_num(xp.exp(reg.x * slope + intercept) * (1 - estimated)) use_mack = xp.sum(w, axis=self.axis, keepdims=True) < 3 if use_mack.any(): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=RuntimeWarning) + # mack_interpolation divides by the square of the sigma two + # periods back. Where that sigma is zero, the quotient is inf or + # nan and the cell's fill comes out as 0. + with np.errstate(divide="ignore", invalid="ignore"): mack_fill = xp.nan_to_num(self.mack_interpolation(y)) * (1 - estimated) fill_ = xp.where(use_mack, mack_fill, fill_) out = xp.nan_to_num(y) + fill_ From 0cbb2d047422ac440b01db2f2aae8712ca2b5973 Mon Sep 17 00:00:00 2001 From: Simon Koch-Sultan Date: Wed, 18 Nov 2026 19:06:39 +0000 Subject: [PATCH 3/4] [FIX] Cap rising log-linear sigma fills after the last positive sigma MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A line through rising sigmas grew the tail fill without bound. Marysville Mut Ins Co othliab IncurLoss has positive sigmas 1.228, 3.872 and 3.569, and the 108-120 fill came out at 107.6. Where the fitted slope is not negative, each fill after the last positive sigma now stays at or below that sigma, the bound Mack's approximation already keeps. Developed with an AI assistant (Árbol, running Muse). --- chainladder/development/tests/test_development.py | 10 ++++++++++ chainladder/utils/weighted_regression.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/chainladder/development/tests/test_development.py b/chainladder/development/tests/test_development.py index fe9fddc04..4f4e6c236 100644 --- a/chainladder/development/tests/test_development.py +++ b/chainladder/development/tests/test_development.py @@ -1163,3 +1163,13 @@ def test_loglinear_sigma_skips_zero_sigmas(clrd): slope, intercept = np.polyfit(period[1:6], np.log(sigma[1:6]), 1) assert np.allclose(sigma[[0, 8]], np.exp(intercept + slope * period[[0, 8]])) assert np.isclose(sigma[0], 7.1819, atol=1e-4) + + +def test_loglinear_sigma_tail_capped_at_last_sigma(clrd): + # GH 1609: Marysville's three positive sigmas (12-24 to 36-48) rise, and + # the line through them filled 107.6 at 108-120. A fill after the last + # positive sigma now stays at or below that sigma, as Mack's fill does. + tri = clrd["IncurLoss"].loc["Marysville Mut Ins Co", "othliab"] + sigma = cl.Development().fit(tri).sigma_.set_backend("numpy").values.ravel() + assert np.isclose(sigma[-1], sigma[2]) + assert np.nanmax(sigma) == np.nanmax(sigma[:3]) diff --git a/chainladder/utils/weighted_regression.py b/chainladder/utils/weighted_regression.py index c2992d7b3..5b77392ff 100644 --- a/chainladder/utils/weighted_regression.py +++ b/chainladder/utils/weighted_regression.py @@ -270,6 +270,10 @@ def loglinear_interpolation(self, y): the log scale that outweighs every other point in the least-squares fit. Zero sigmas are left as they are. + Where the fitted line rises, a fill after the last positive sigma is + capped at that sigma, so the tail sigmas never rise above the last one + observed. + With fewer than three positive sigmas, the fill comes from ``mack_interpolation`` instead. One point gives no slope, and a line through two points fits both exactly, so no residual is left to judge @@ -284,6 +288,17 @@ def loglinear_interpolation(self, y): reg = WeightedRegression(self.axis, False, xp=xp).fit(None, ly, w) slope, intercept = reg.slope_, reg.intercept_ fill_ = xp.nan_to_num(xp.exp(reg.x * slope + intercept) * (1 - estimated)) + # Where the fitted line rises, a fill after the last positive sigma + # stays at or below that sigma, as each Mack fill stays at or below + # the sigmas before it. A rising line would otherwise grow the tail + # sigmas without bound. + last_x = xp.max(reg.x * w, axis=self.axis, keepdims=True) + last_sigma = xp.sum( + xp.nan_to_num(y) * (reg.x == last_x), axis=self.axis, keepdims=True + ) + fill_ = xp.where( + (reg.x > last_x) & (slope >= 0), xp.minimum(fill_, last_sigma), fill_ + ) use_mack = xp.sum(w, axis=self.axis, keepdims=True) < 3 if use_mack.any(): # mack_interpolation divides by the square of the sigma two From 327ee23f8f908cc95b37eebe5259708760ce17c2 Mon Sep 17 00:00:00 2001 From: Simon Koch-Sultan Date: Thu, 19 Nov 2026 03:05:06 +0000 Subject: [PATCH 4/4] [FIX] Leave sigmas before the first positive sigma unfilled and drop 1e-320 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The log-linear fill extended its line to ages before the first positive sigma. That is extrapolation, and Mack's approximation never fills those cells. The fill now leaves them unfilled, so they stay NaN under both sigma_interpolation settings. IL State Bar's CumPaidLoss 24-36 sigma goes from 3,341,035 to NaN. mack_interpolation no longer swaps zero sigmas for 1e-320. The fills it makes do not change, and zero sigmas now come out as NaN, as they already do under log-linear. Developed with an AI assistant (Árbol, running Muse). --- .../development/tests/test_development.py | 21 +++++++++++-- chainladder/utils/weighted_regression.py | 31 +++++++++++-------- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/chainladder/development/tests/test_development.py b/chainladder/development/tests/test_development.py index 4f4e6c236..aeaae644b 100644 --- a/chainladder/development/tests/test_development.py +++ b/chainladder/development/tests/test_development.py @@ -1154,15 +1154,18 @@ def test_loglinear_sigma_skips_zero_sigmas(clrd): fill = loglinear.values.ravel()[2] assert np.isclose(fill, mack.sigma_.set_backend("numpy").values.ravel()[2]) assert np.isclose(fill, 14.9781, atol=1e-4) + # Both paths report the zero sigmas after 36-48 as NaN, with no 1e-320. + assert np.isnan(mack.sigma_.set_backend("numpy").values.ravel()[3:]).all() # Five positive sigmas (24-36 to 72-84) and two zeros (84-96, 96-108): - # the 12-24 and 108-120 fills come from the five positive sigmas only. + # the 108-120 fill comes from the five positive sigmas only, and 12-24, + # before the first positive sigma, stays unfilled. buckeye = cl.Development().fit(tri.loc["Buckeye Ins Grp", "othliab"]) sigma = buckeye.sigma_.set_backend("numpy").values.ravel() period = np.arange(1, sigma.size + 1) slope, intercept = np.polyfit(period[1:6], np.log(sigma[1:6]), 1) - assert np.allclose(sigma[[0, 8]], np.exp(intercept + slope * period[[0, 8]])) - assert np.isclose(sigma[0], 7.1819, atol=1e-4) + assert np.isclose(sigma[8], np.exp(intercept + slope * period[8])) + assert np.isnan(sigma[0]) def test_loglinear_sigma_tail_capped_at_last_sigma(clrd): @@ -1173,3 +1176,15 @@ def test_loglinear_sigma_tail_capped_at_last_sigma(clrd): sigma = cl.Development().fit(tri).sigma_.set_backend("numpy").values.ravel() assert np.isclose(sigma[-1], sigma[2]) assert np.nanmax(sigma) == np.nanmax(sigma[:3]) + + +def test_loglinear_sigma_not_filled_before_first_positive(clrd): + # GH 1609: IL State Bar's positive sigmas start at 60-72, and the line + # through them filled 3,341,035 at 24-36. Mack's approximation leaves + # that cell unfilled, and so does the log-linear fill now. + tri = clrd["CumPaidLoss"].loc["IL State Bar Assoc Mut Ins Co", "othliab"] + loglinear = cl.Development().fit(tri).sigma_.set_backend("numpy").values.ravel() + mack = cl.Development(sigma_interpolation="mack").fit(tri) + mack = mack.sigma_.set_backend("numpy").values.ravel() + assert np.isnan(loglinear[1]) and np.isnan(mack[1]) + assert np.nanmax(loglinear) < 15 diff --git a/chainladder/utils/weighted_regression.py b/chainladder/utils/weighted_regression.py index 5b77392ff..5493194a2 100644 --- a/chainladder/utils/weighted_regression.py +++ b/chainladder/utils/weighted_regression.py @@ -266,13 +266,14 @@ def loglinear_interpolation(self, y): Fill undefined sigmas by regressing log(sigma) on development period. Only positive sigmas enter the regression. A zero sigma has no - logarithm, and a stand-in such as 1e-320 puts a point near -737 on - the log scale that outweighs every other point in the least-squares - fit. Zero sigmas are left as they are. + logarithm, so it stays out of the fit and is left as it is. - Where the fitted line rises, a fill after the last positive sigma is - capped at that sigma, so the tail sigmas never rise above the last one - observed. + Sigmas before the first positive sigma stay unfilled. A fill there + would extend the line past the data, and Mack's approximation never + fills those cells either. Between the first and last positive sigma + the fill interpolates. After the last one, where the fitted line + rises, a fill is capped at that sigma, so the tail sigmas never rise + above the last one observed. With fewer than three positive sigmas, the fill comes from ``mack_interpolation`` instead. One point gives no slope, and a line @@ -307,6 +308,8 @@ def loglinear_interpolation(self, y): with np.errstate(divide="ignore", invalid="ignore"): mack_fill = xp.nan_to_num(self.mack_interpolation(y)) * (1 - estimated) fill_ = xp.where(use_mack, mack_fill, fill_) + first_x = xp.min(xp.where(w > 0, reg.x, xp.inf), axis=self.axis, keepdims=True) + fill_ = xp.where(reg.x < first_x, 0, fill_) out = xp.nan_to_num(y) + fill_ return num_to_nan(out) @@ -320,7 +323,6 @@ def mack_interpolation(self, y): xp = self.xp ly = y.copy() - ly = xp.where(ly == 0, 1e-320, ly) w = xp.nan_to_num(ly * 0 + 1) slicer_n, slicer_d, slicer_a = ( ([slice(None)] * 4), @@ -337,14 +339,17 @@ def mack_interpolation(self, y): tuple(slicer_d), tuple(slicer_a), ) - fill_ = xp.sqrt( - abs( - xp.minimum( - (ly[slicer_n] ** 4 / ly[slicer_d] ** 2), - xp.minimum(ly[slicer_d] ** 2, ly[slicer_n] ** 2), + # A zero sigma two periods back makes the quotient inf or nan, and + # nan_to_num below turns that cell's fill into 0. + with np.errstate(divide="ignore", invalid="ignore"): + fill_ = xp.sqrt( + abs( + xp.minimum( + (ly[slicer_n] ** 4 / ly[slicer_d] ** 2), + xp.minimum(ly[slicer_d] ** 2, ly[slicer_n] ** 2), + ) ) ) - ) fill_ = xp.concatenate((w[slicer_a], xp.nan_to_num(fill_)), axis=self.axis) * ( 1 - w )