Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 56 additions & 0 deletions chainladder/development/tests/test_development.py
Original file line number Diff line number Diff line change
Expand Up @@ -1132,3 +1132,59 @@ def test_valid_params_pass_validation(raa):
cl.Development(average=lambda age: "geometric", n_periods=[-1] * 9).fit(raa)
dev = cl.Development(n_periods=4, average="simple")
assert cl.read_json(dev.to_json()).get_params() == dev.get_params()


def test_loglinear_sigma_skips_zero_sigmas(clrd):
# GH 1609: zero sigmas used to enter the log-linear fit as log(1e-320).
tri = clrd["CumPaidLoss"]

# One positive sigma (36-48) and a zero (48-60): no line is fitted, and
# Mack's approximation has no two positive sigmas in a row, so the
# 12-24 and 24-36 sigmas stay NaN instead of 1.8e308.
brotherhood = cl.Development().fit(tri.loc["Brotherhood Mut Ins Co", "prodliab"])
assert np.nanmax(brotherhood.sigma_.set_backend("numpy").values) < 1e3

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please double check my logic flow here.

we are here to test mack's approximation for sigma_ when loglinear regression doesn't have enough valid data. have we tested mack's approximatio itself somewhere? i.e. are any of our hardcoded mack tests covering the approximation for sigma?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Your logic is right, and the Brotherhood case didn't test it. Its raw sigmas are NaN, NaN, 7.67, 0, then NaN. Mack's approximation needs two positive sigmas just before a gap, so it fills nothing there; the case only checks that the 1.8e308 fills are gone. I fixed its comment and added Great Amer Grp / medmal, whose sigmas are 14.98, 52.90, then a gap: the fallback fills 36-48 with 14.978, and the test checks that against sigma_interpolation="mack" on the same triangle. On main that fill is 1.2e-142. The approximation itself is covered by printed values: test_mack1999_hardcode checks mortgage's filled last sigma (16.887 against 16.89 in Mack 1999), and test_cdr_matches_merz2008 checks mw2008's filled last sigma^2 (0.040 against 0.04). test_mack1994_hardcode only reaches it through the oldest origin's standard error.

(This comment was written with Árbol, an AI assistant, under my direction.)

assert np.isfinite(brotherhood.std_err_.set_backend("numpy").values).all()

# Two positive sigmas (12-24, 24-36), then none at 36-48 and zeros after:
# the 36-48 fill comes from Mack's approximation, the same value as
# under sigma_interpolation="mack".
great_amer = tri.loc["Great Amer Grp", "medmal"]
loglinear = cl.Development().fit(great_amer).sigma_.set_backend("numpy")
mack = cl.Development(sigma_interpolation="mack").fit(great_amer)
fill = loglinear.values.ravel()[2]
assert np.isclose(fill, mack.sigma_.set_backend("numpy").values.ravel()[2])
assert np.isclose(fill, 14.9781, atol=1e-4)
# Both paths report the zero sigmas after 36-48 as NaN, with no 1e-320.
assert np.isnan(mack.sigma_.set_backend("numpy").values.ravel()[3:]).all()

# Five positive sigmas (24-36 to 72-84) and two zeros (84-96, 96-108):
# the 108-120 fill comes from the five positive sigmas only, and 12-24,
# before the first positive sigma, stays unfilled.
buckeye = cl.Development().fit(tri.loc["Buckeye Ins Grp", "othliab"])
sigma = buckeye.sigma_.set_backend("numpy").values.ravel()
period = np.arange(1, sigma.size + 1)
slope, intercept = np.polyfit(period[1:6], np.log(sigma[1:6]), 1)
assert np.isclose(sigma[8], np.exp(intercept + slope * period[8]))
assert np.isnan(sigma[0])


def test_loglinear_sigma_tail_capped_at_last_sigma(clrd):
# GH 1609: Marysville's three positive sigmas (12-24 to 36-48) rise, and
# the line through them filled 107.6 at 108-120. A fill after the last
# positive sigma now stays at or below that sigma, as Mack's fill does.
tri = clrd["IncurLoss"].loc["Marysville Mut Ins Co", "othliab"]
sigma = cl.Development().fit(tri).sigma_.set_backend("numpy").values.ravel()
assert np.isclose(sigma[-1], sigma[2])
assert np.nanmax(sigma) == np.nanmax(sigma[:3])


def test_loglinear_sigma_not_filled_before_first_positive(clrd):
# GH 1609: IL State Bar's positive sigmas start at 60-72, and the line
# through them filled 3,341,035 at 24-36. Mack's approximation leaves
# that cell unfilled, and so does the log-linear fill now.
tri = clrd["CumPaidLoss"].loc["IL State Bar Assoc Mut Ins Co", "othliab"]
loglinear = cl.Development().fit(tri).sigma_.set_backend("numpy").values.ravel()
mack = cl.Development(sigma_interpolation="mack").fit(tri)
mack = mack.sigma_.set_backend("numpy").values.ravel()
assert np.isnan(loglinear[1]) and np.isnan(mack[1])
assert np.nanmax(loglinear) < 15
64 changes: 52 additions & 12 deletions chainladder/utils/weighted_regression.py
Original file line number Diff line number Diff line change
Expand Up @@ -262,17 +262,55 @@ def std_err_fill(self):
return self

def loglinear_interpolation(self, y):
"""Use Cases: generally for filling in last element of sigma_"""
"""
Fill undefined sigmas by regressing log(sigma) on development period.

Only positive sigmas enter the regression. A zero sigma has no
logarithm, so it stays out of the fit and is left as it is.

Sigmas before the first positive sigma stay unfilled. A fill there
would extend the line past the data, and Mack's approximation never
fills those cells either. Between the first and last positive sigma
the fill interpolates. After the last one, where the fitted line
rises, a fill is capped at that sigma, so the tail sigmas never rise
above the last one observed.

With fewer than three positive sigmas, the fill comes from
``mack_interpolation`` instead. One point gives no slope, and a line
through two points fits both exactly, so no residual is left to judge
it by.
"""
from chainladder.utils.utility_functions import num_to_nan

xp = self.xp
ly = y.copy()
ly = xp.log(xp.where(ly == 0, 1e-320, ly))
ly = xp.log(xp.where(y > 0, y, xp.nan))
w = xp.nan_to_num(ly * 0 + 1)
estimated = xp.nan_to_num(y * 0 + 1)
reg = WeightedRegression(self.axis, False, xp=xp).fit(None, ly, w)
slope, intercept = reg.slope_, reg.intercept_
fill_ = xp.exp(reg.x * slope + intercept) * (1 - w)
out = xp.nan_to_num(y) + xp.nan_to_num(fill_)
fill_ = xp.nan_to_num(xp.exp(reg.x * slope + intercept) * (1 - estimated))
# Where the fitted line rises, a fill after the last positive sigma
# stays at or below that sigma, as each Mack fill stays at or below
# the sigmas before it. A rising line would otherwise grow the tail
# sigmas without bound.
last_x = xp.max(reg.x * w, axis=self.axis, keepdims=True)
last_sigma = xp.sum(
xp.nan_to_num(y) * (reg.x == last_x), axis=self.axis, keepdims=True
)
fill_ = xp.where(
(reg.x > last_x) & (slope >= 0), xp.minimum(fill_, last_sigma), fill_
)
use_mack = xp.sum(w, axis=self.axis, keepdims=True) < 3
if use_mack.any():
# mack_interpolation divides by the square of the sigma two
# periods back. Where that sigma is zero, the quotient is inf or
# nan and the cell's fill comes out as 0.
with np.errstate(divide="ignore", invalid="ignore"):
mack_fill = xp.nan_to_num(self.mack_interpolation(y)) * (1 - estimated)
fill_ = xp.where(use_mack, mack_fill, fill_)
first_x = xp.min(xp.where(w > 0, reg.x, xp.inf), axis=self.axis, keepdims=True)
fill_ = xp.where(reg.x < first_x, 0, fill_)
out = xp.nan_to_num(y) + fill_
return num_to_nan(out)

def mack_interpolation(self, y):
Expand All @@ -285,7 +323,6 @@ def mack_interpolation(self, y):

xp = self.xp
ly = y.copy()
ly = xp.where(ly == 0, 1e-320, ly)
w = xp.nan_to_num(ly * 0 + 1)
slicer_n, slicer_d, slicer_a = (
([slice(None)] * 4),
Expand All @@ -302,14 +339,17 @@ def mack_interpolation(self, y):
tuple(slicer_d),
tuple(slicer_a),
)
fill_ = xp.sqrt(
abs(
xp.minimum(
(ly[slicer_n] ** 4 / ly[slicer_d] ** 2),
xp.minimum(ly[slicer_d] ** 2, ly[slicer_n] ** 2),
# A zero sigma two periods back makes the quotient inf or nan, and
# nan_to_num below turns that cell's fill into 0.
with np.errstate(divide="ignore", invalid="ignore"):
fill_ = xp.sqrt(
abs(
xp.minimum(
(ly[slicer_n] ** 4 / ly[slicer_d] ** 2),
xp.minimum(ly[slicer_d] ** 2, ly[slicer_n] ** 2),
)
)
)
)
fill_ = xp.concatenate((w[slicer_a], xp.nan_to_num(fill_)), axis=self.axis) * (
1 - w
)
Expand Down