diff --git a/chainladder/development/learning.py b/chainladder/development/learning.py index c809f49bb..d4b071505 100644 --- a/chainladder/development/learning.py +++ b/chainladder/development/learning.py @@ -208,17 +208,24 @@ def _get_triangle_ml(self, df, preds=None): X_r = [df] y_r = [preds] dgrain = __grain_to_months__[self.development_grain_] - ograin = 12 // __grain_to_months__[self.origin_grain_] - latest_filter = (df["origin"] + 1) * ograin + ( - df["development"] - dgrain - ) / dgrain - latest_filter = latest_filter == latest_filter.max() + # The latest diagonal holds the cells at the latest valuation. Comparing + # encoded valuations finds it whether or not the origin and development + # grains match, as in OYDM. + latest_filter = df["valuation"] == df["valuation"].max() preds = pd.DataFrame(preds.copy())[latest_filter].values out = df.loc[latest_filter].copy() + # valuation_encoder_ maps the i-th valuation date to i / periods_per_year. + # Step the integer index i and divide once so that projected valuations + # match the encoder's values exactly at every grain. + periods_per_year = 12 // dgrain + val_index = { + v: i for i, v in enumerate(sorted(self.valuation_encoder_.values())) + } + latest_val_index = out["valuation"].map(val_index) dev_lags = df["development"].drop_duplicates().sort_values() - for d in dev_lags[1:]: + for step, d in enumerate(dev_lags[1:], start=1): out["development"] = out["development"] + dgrain - out["valuation"] = out["valuation"] + dgrain / 12 + out["valuation"] = (latest_val_index + step) / periods_per_year if len(preds.shape) == 1: preds = preds[:, None] if self.autoregressive: diff --git a/chainladder/development/tests/test_learning.py b/chainladder/development/tests/test_learning.py index a496393e1..952dd53a4 100644 --- a/chainladder/development/tests/test_learning.py +++ b/chainladder/development/tests/test_learning.py @@ -1,8 +1,10 @@ import chainladder as cl -from sklearn.linear_model import LinearRegression +import pytest +from sklearn.linear_model import LinearRegression, TweedieRegressor from sklearn.pipeline import Pipeline from chainladder.utils.utility_functions import PatsyFormula import pandas as pd +import numpy as np def test_incremental(genins): @@ -74,3 +76,98 @@ def test_grain(): ).fit(tri) assert grains[period] == len(model.triangle_ml_.development) assert grains[period] == len(model.triangle_ml_.origin) + assert not np.isnan(model.triangle_ml_.values).any() + + +GRAINS = [ + "OMDM", + "OQDM", + "OSDM", + "OYDM", + "OQDQ", + "OSDQ", + "OYDQ", + "OSDS", + "OYDS", + "OYDY", +] + + +@pytest.fixture +def exact_monthly(): + """ + Two years of monthly origins whose cumulative values follow one + multiplicative development pattern, so every grain has at least two + origins and a full upper triangle. + """ + n = 24 + origins = pd.date_range("2020-01-01", periods=n, freq="MS") + pattern = 1 - 0.5 ** np.arange(1, n + 1) + rows = [ + { + "origin": o, + "valuation": o + pd.DateOffset(months=k + 1) - pd.Timedelta(days=1), + "values": 1000.0 * (1 + 0.1 * a) * pattern[k], + } + for a, o in enumerate(origins) + for k in range(n - a) + ] + return cl.Triangle( + pd.DataFrame(rows), + origin="origin", + valuation="valuation", + columns="values", + cumulative=True, + ) + + +# newton-cholesky converges in about ten iterations here, where lbfgs needs +# thousands. Its gradient step emits a matmul RuntimeWarning, and the fit +# still matches chain ladder at the tolerance below. +@pytest.mark.filterwarnings("ignore:invalid value encountered in matmul:RuntimeWarning") +@pytest.mark.parametrize("grain", GRAINS) +def test_poisson_glm_matches_chainladder(exact_monthly, grain): + """ + A cross-classified Poisson GLM reproduces chain ladder (Renshaw and + Verrall, 1998, renshaw1998 in docs/library/references.bib), so at every + grain the projected square must be complete and its chain ladder + ultimates must equal the ultimates of chain ladder on the triangle. + """ + tri = exact_monthly.grain(grain) + model = cl.DevelopmentML( + Pipeline( + steps=[ + ("design_matrix", PatsyFormula("C(development) + C(origin)")), + ( + "model", + TweedieRegressor( + power=1, + link="log", + fit_intercept=False, + alpha=0, + solver="newton-cholesky", + ), + ), + ] + ), + y_ml="values", + ).fit(tri) + assert not np.isnan(model.triangle_ml_.values).any() + expected = cl.Chainladder().fit(tri).ultimate_.values + actual = cl.Chainladder().fit(model.transform(tri)).ultimate_.values + assert np.allclose(actual, expected, rtol=1e-4) + + +@pytest.mark.parametrize("grain", GRAINS) +def test_projection_fills_future_cells(monthly, grain): + """ + DevelopmentML projects each origin forward from the latest diagonal, so + every future cell of triangle_ml_ must be filled at every grain, + including grains whose origin and development periods differ. + """ + tri = monthly["reportedCount"].grain(grain) + ml = cl.TweedieGLM(response="reportedCount").fit(tri).triangle_ml_ + values = ml.set_backend("numpy").values[0, 0] + future = np.isnan(tri.set_backend("numpy").nan_triangle) + assert values.shape == future.shape + assert not np.isnan(values[future]).any() diff --git a/docs/library/references.bib b/docs/library/references.bib index c3ca2ff67..e5ec9ce43 100644 --- a/docs/library/references.bib +++ b/docs/library/references.bib @@ -120,3 +120,14 @@ @article{merz2014 year = {2014}, url = {https://doi.org/10.2139/ssrn.2524352} } + +@article{renshaw1998, + author = {Renshaw, A. E. and Verrall, R. J.}, + title = {A {S}tochastic {M}odel {U}nderlying the {C}hain-{L}adder {T}echnique}, + journal = {British Actuarial Journal}, + volume = {4}, + number = {4}, + pages = {903--923}, + year = {1998}, + url = {https://doi.org/10.1017/S1357321700000222} +}