diff --git a/chainladder/development/tests/test_development.py b/chainladder/development/tests/test_development.py index c6e8b78b9..aeaae644b 100644 --- a/chainladder/development/tests/test_development.py +++ b/chainladder/development/tests/test_development.py @@ -1132,3 +1132,59 @@ def test_valid_params_pass_validation(raa): cl.Development(average=lambda age: "geometric", n_periods=[-1] * 9).fit(raa) dev = cl.Development(n_periods=4, average="simple") assert cl.read_json(dev.to_json()).get_params() == dev.get_params() + + +def test_loglinear_sigma_skips_zero_sigmas(clrd): + # GH 1609: zero sigmas used to enter the log-linear fit as log(1e-320). + tri = clrd["CumPaidLoss"] + + # One positive sigma (36-48) and a zero (48-60): no line is fitted, and + # Mack's approximation has no two positive sigmas in a row, so the + # 12-24 and 24-36 sigmas stay NaN instead of 1.8e308. + brotherhood = cl.Development().fit(tri.loc["Brotherhood Mut Ins Co", "prodliab"]) + assert np.nanmax(brotherhood.sigma_.set_backend("numpy").values) < 1e3 + assert np.isfinite(brotherhood.std_err_.set_backend("numpy").values).all() + + # Two positive sigmas (12-24, 24-36), then none at 36-48 and zeros after: + # the 36-48 fill comes from Mack's approximation, the same value as + # under sigma_interpolation="mack". + great_amer = tri.loc["Great Amer Grp", "medmal"] + loglinear = cl.Development().fit(great_amer).sigma_.set_backend("numpy") + mack = cl.Development(sigma_interpolation="mack").fit(great_amer) + fill = loglinear.values.ravel()[2] + assert np.isclose(fill, mack.sigma_.set_backend("numpy").values.ravel()[2]) + assert np.isclose(fill, 14.9781, atol=1e-4) + # Both paths report the zero sigmas after 36-48 as NaN, with no 1e-320. + assert np.isnan(mack.sigma_.set_backend("numpy").values.ravel()[3:]).all() + + # Five positive sigmas (24-36 to 72-84) and two zeros (84-96, 96-108): + # the 108-120 fill comes from the five positive sigmas only, and 12-24, + # before the first positive sigma, stays unfilled. + buckeye = cl.Development().fit(tri.loc["Buckeye Ins Grp", "othliab"]) + sigma = buckeye.sigma_.set_backend("numpy").values.ravel() + period = np.arange(1, sigma.size + 1) + slope, intercept = np.polyfit(period[1:6], np.log(sigma[1:6]), 1) + assert np.isclose(sigma[8], np.exp(intercept + slope * period[8])) + assert np.isnan(sigma[0]) + + +def test_loglinear_sigma_tail_capped_at_last_sigma(clrd): + # GH 1609: Marysville's three positive sigmas (12-24 to 36-48) rise, and + # the line through them filled 107.6 at 108-120. A fill after the last + # positive sigma now stays at or below that sigma, as Mack's fill does. + tri = clrd["IncurLoss"].loc["Marysville Mut Ins Co", "othliab"] + sigma = cl.Development().fit(tri).sigma_.set_backend("numpy").values.ravel() + assert np.isclose(sigma[-1], sigma[2]) + assert np.nanmax(sigma) == np.nanmax(sigma[:3]) + + +def test_loglinear_sigma_not_filled_before_first_positive(clrd): + # GH 1609: IL State Bar's positive sigmas start at 60-72, and the line + # through them filled 3,341,035 at 24-36. Mack's approximation leaves + # that cell unfilled, and so does the log-linear fill now. + tri = clrd["CumPaidLoss"].loc["IL State Bar Assoc Mut Ins Co", "othliab"] + loglinear = cl.Development().fit(tri).sigma_.set_backend("numpy").values.ravel() + mack = cl.Development(sigma_interpolation="mack").fit(tri) + mack = mack.sigma_.set_backend("numpy").values.ravel() + assert np.isnan(loglinear[1]) and np.isnan(mack[1]) + assert np.nanmax(loglinear) < 15 diff --git a/chainladder/utils/weighted_regression.py b/chainladder/utils/weighted_regression.py index 5e0a51a4b..5493194a2 100644 --- a/chainladder/utils/weighted_regression.py +++ b/chainladder/utils/weighted_regression.py @@ -262,17 +262,55 @@ def std_err_fill(self): return self def loglinear_interpolation(self, y): - """Use Cases: generally for filling in last element of sigma_""" + """ + Fill undefined sigmas by regressing log(sigma) on development period. + + Only positive sigmas enter the regression. A zero sigma has no + logarithm, so it stays out of the fit and is left as it is. + + Sigmas before the first positive sigma stay unfilled. A fill there + would extend the line past the data, and Mack's approximation never + fills those cells either. Between the first and last positive sigma + the fill interpolates. After the last one, where the fitted line + rises, a fill is capped at that sigma, so the tail sigmas never rise + above the last one observed. + + With fewer than three positive sigmas, the fill comes from + ``mack_interpolation`` instead. One point gives no slope, and a line + through two points fits both exactly, so no residual is left to judge + it by. + """ from chainladder.utils.utility_functions import num_to_nan xp = self.xp - ly = y.copy() - ly = xp.log(xp.where(ly == 0, 1e-320, ly)) + ly = xp.log(xp.where(y > 0, y, xp.nan)) w = xp.nan_to_num(ly * 0 + 1) + estimated = xp.nan_to_num(y * 0 + 1) reg = WeightedRegression(self.axis, False, xp=xp).fit(None, ly, w) slope, intercept = reg.slope_, reg.intercept_ - fill_ = xp.exp(reg.x * slope + intercept) * (1 - w) - out = xp.nan_to_num(y) + xp.nan_to_num(fill_) + fill_ = xp.nan_to_num(xp.exp(reg.x * slope + intercept) * (1 - estimated)) + # Where the fitted line rises, a fill after the last positive sigma + # stays at or below that sigma, as each Mack fill stays at or below + # the sigmas before it. A rising line would otherwise grow the tail + # sigmas without bound. + last_x = xp.max(reg.x * w, axis=self.axis, keepdims=True) + last_sigma = xp.sum( + xp.nan_to_num(y) * (reg.x == last_x), axis=self.axis, keepdims=True + ) + fill_ = xp.where( + (reg.x > last_x) & (slope >= 0), xp.minimum(fill_, last_sigma), fill_ + ) + use_mack = xp.sum(w, axis=self.axis, keepdims=True) < 3 + if use_mack.any(): + # mack_interpolation divides by the square of the sigma two + # periods back. Where that sigma is zero, the quotient is inf or + # nan and the cell's fill comes out as 0. + with np.errstate(divide="ignore", invalid="ignore"): + mack_fill = xp.nan_to_num(self.mack_interpolation(y)) * (1 - estimated) + fill_ = xp.where(use_mack, mack_fill, fill_) + first_x = xp.min(xp.where(w > 0, reg.x, xp.inf), axis=self.axis, keepdims=True) + fill_ = xp.where(reg.x < first_x, 0, fill_) + out = xp.nan_to_num(y) + fill_ return num_to_nan(out) def mack_interpolation(self, y): @@ -285,7 +323,6 @@ def mack_interpolation(self, y): xp = self.xp ly = y.copy() - ly = xp.where(ly == 0, 1e-320, ly) w = xp.nan_to_num(ly * 0 + 1) slicer_n, slicer_d, slicer_a = ( ([slice(None)] * 4), @@ -302,14 +339,17 @@ def mack_interpolation(self, y): tuple(slicer_d), tuple(slicer_a), ) - fill_ = xp.sqrt( - abs( - xp.minimum( - (ly[slicer_n] ** 4 / ly[slicer_d] ** 2), - xp.minimum(ly[slicer_d] ** 2, ly[slicer_n] ** 2), + # A zero sigma two periods back makes the quotient inf or nan, and + # nan_to_num below turns that cell's fill into 0. + with np.errstate(divide="ignore", invalid="ignore"): + fill_ = xp.sqrt( + abs( + xp.minimum( + (ly[slicer_n] ** 4 / ly[slicer_d] ** 2), + xp.minimum(ly[slicer_d] ** 2, ly[slicer_n] ** 2), + ) ) ) - ) fill_ = xp.concatenate((w[slicer_a], xp.nan_to_num(fill_)), axis=self.axis) * ( 1 - w )