Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 14 additions & 7 deletions chainladder/development/learning.py
Original file line number Diff line number Diff line change
Expand Up @@ -208,17 +208,24 @@ def _get_triangle_ml(self, df, preds=None):
X_r = [df]
y_r = [preds]
dgrain = __grain_to_months__[self.development_grain_]
ograin = 12 // __grain_to_months__[self.origin_grain_]
latest_filter = (df["origin"] + 1) * ograin + (
df["development"] - dgrain
) / dgrain
latest_filter = latest_filter == latest_filter.max()
# The latest diagonal holds the cells at the latest valuation. Comparing
# encoded valuations finds it whether or not the origin and development
# grains match, as in OYDM.
latest_filter = df["valuation"] == df["valuation"].max()
preds = pd.DataFrame(preds.copy())[latest_filter].values
out = df.loc[latest_filter].copy()
# valuation_encoder_ maps the i-th valuation date to i / periods_per_year.
# Step the integer index i and divide once so that projected valuations
# match the encoder's values exactly at every grain.
periods_per_year = 12 // dgrain
val_index = {
v: i for i, v in enumerate(sorted(self.valuation_encoder_.values()))
}
latest_val_index = out["valuation"].map(val_index)
dev_lags = df["development"].drop_duplicates().sort_values()
for d in dev_lags[1:]:
for step, d in enumerate(dev_lags[1:], start=1):
out["development"] = out["development"] + dgrain
out["valuation"] = out["valuation"] + dgrain / 12
out["valuation"] = (latest_val_index + step) / periods_per_year
if len(preds.shape) == 1:
preds = preds[:, None]
if self.autoregressive:
Expand Down
99 changes: 98 additions & 1 deletion chainladder/development/tests/test_learning.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import chainladder as cl
from sklearn.linear_model import LinearRegression
import pytest
from sklearn.linear_model import LinearRegression, TweedieRegressor
from sklearn.pipeline import Pipeline
from chainladder.utils.utility_functions import PatsyFormula
import pandas as pd
import numpy as np


def test_incremental(genins):
Expand Down Expand Up @@ -74,3 +76,98 @@ def test_grain():
).fit(tri)
assert grains[period] == len(model.triangle_ml_.development)
assert grains[period] == len(model.triangle_ml_.origin)
assert not np.isnan(model.triangle_ml_.values).any()


GRAINS = [
"OMDM",
"OQDM",
"OSDM",
"OYDM",
"OQDQ",
"OSDQ",
"OYDQ",
"OSDS",
"OYDS",
"OYDY",
]


@pytest.fixture
def exact_monthly():
"""
Two years of monthly origins whose cumulative values follow one
multiplicative development pattern, so every grain has at least two
origins and a full upper triangle.
"""
n = 24
origins = pd.date_range("2020-01-01", periods=n, freq="MS")
pattern = 1 - 0.5 ** np.arange(1, n + 1)
rows = [
{
"origin": o,
"valuation": o + pd.DateOffset(months=k + 1) - pd.Timedelta(days=1),
"values": 1000.0 * (1 + 0.1 * a) * pattern[k],
}
for a, o in enumerate(origins)
for k in range(n - a)
]
return cl.Triangle(
pd.DataFrame(rows),
origin="origin",
valuation="valuation",
columns="values",
cumulative=True,
)


# newton-cholesky converges in about ten iterations here, where lbfgs needs
# thousands. Its gradient step emits a matmul RuntimeWarning, and the fit
# still matches chain ladder at the tolerance below.
@pytest.mark.filterwarnings("ignore:invalid value encountered in matmul:RuntimeWarning")
@pytest.mark.parametrize("grain", GRAINS)
def test_poisson_glm_matches_chainladder(exact_monthly, grain):
"""
A cross-classified Poisson GLM reproduces chain ladder (Renshaw and
Verrall, 1998, renshaw1998 in docs/library/references.bib), so at every
grain the projected square must be complete and its chain ladder
ultimates must equal the ultimates of chain ladder on the triangle.
"""
tri = exact_monthly.grain(grain)
model = cl.DevelopmentML(
Pipeline(
steps=[
("design_matrix", PatsyFormula("C(development) + C(origin)")),
(
"model",
TweedieRegressor(
power=1,
link="log",
fit_intercept=False,
alpha=0,
solver="newton-cholesky",
),
),
]
),
y_ml="values",
).fit(tri)
assert not np.isnan(model.triangle_ml_.values).any()
expected = cl.Chainladder().fit(tri).ultimate_.values
actual = cl.Chainladder().fit(model.transform(tri)).ultimate_.values
assert np.allclose(actual, expected, rtol=1e-4)


@pytest.mark.parametrize("grain", GRAINS)
def test_projection_fills_future_cells(monthly, grain):

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if you fixturized the data from test_monthly_poisson_glm_matches_chainladder, we could have a stronger test that pins the actual ultimates through all the grains?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done. The exact-pattern data is now the exact_monthly fixture, extended from 8 to 24 monthly origins so the yearly grains keep two origins. With 8, OYDY collapses to a single cell. test_poisson_glm_matches_chainladder runs at all ten grains and checks the GLM's chain ladder ultimates against cl.Chainladder on the same triangle. On main it fails at seven of the ten, every grain except OQDQ, OSDS and OYDY, with NaN left in triangle_ml_ and a lower total. I kept test_projection_fills_future_cells on the monthly sample since it covers the TweedieGLM wrapper on real data. The new test fits with solver="newton-cholesky", which converges in about ten iterations. With lbfgs at max_iter=1000, three grains stopped at the cap with a ConvergenceWarning, and raising the cap until they converge took the file from about 3 to 6 minutes. The test filters the matmul RuntimeWarning that newton-cholesky emits, and the ultimates still match at rtol=1e-4.

(This comment was written with Árbol, an AI assistant, under my direction.)

"""
DevelopmentML projects each origin forward from the latest diagonal, so
every future cell of triangle_ml_ must be filled at every grain,
including grains whose origin and development periods differ.
"""
tri = monthly["reportedCount"].grain(grain)
ml = cl.TweedieGLM(response="reportedCount").fit(tri).triangle_ml_
values = ml.set_backend("numpy").values[0, 0]
future = np.isnan(tri.set_backend("numpy").nan_triangle)
assert values.shape == future.shape
assert not np.isnan(values[future]).any()
11 changes: 11 additions & 0 deletions docs/library/references.bib
Original file line number Diff line number Diff line change
Expand Up @@ -120,3 +120,14 @@ @article{merz2014
year = {2014},
url = {https://doi.org/10.2139/ssrn.2524352}
}

@article{renshaw1998,
author = {Renshaw, A. E. and Verrall, R. J.},
title = {A {S}tochastic {M}odel {U}nderlying the {C}hain-{L}adder {T}echnique},
journal = {British Actuarial Journal},
volume = {4},
number = {4},
pages = {903--923},
year = {1998},
url = {https://doi.org/10.1017/S1357321700000222}
}
Loading