diff --git a/.gitignore b/.gitignore index 7808b45c4..f459162f0 100644 --- a/.gitignore +++ b/.gitignore @@ -117,6 +117,8 @@ settings.json *.pkl +!chainladder/utils/data/*.pkl + # macOS related files *.DS_Store *Icon? diff --git a/chainladder/core/tests/test_io.py b/chainladder/core/tests/test_io.py new file mode 100644 index 000000000..b9b56d6e6 --- /dev/null +++ b/chainladder/core/tests/test_io.py @@ -0,0 +1,263 @@ +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +import dill +import numpy as np +import pytest + +import chainladder as cl + +if TYPE_CHECKING: + from chainladder import Triangle + + +class TestJSON: + """Test JSON IO""" + + def test_triangle_json_io(self, clrd: Triangle) -> None: + """ + test JSON round trip for Triangle + + Parameters + ---------- + clrd: Triangle + The CLRD sample data set fixture + + Returns + ------- + None + """ + clrd2 = cl.read_json(clrd.to_json(), array_backend=clrd.array_backend) + assert clrd.equals(clrd2) + assert clrd.index.equals(clrd2.index) + assert np.all(clrd.columns == clrd2.columns) + assert np.all(clrd.odims == clrd2.odims) + assert np.all(clrd.ddims == clrd2.ddims) + assert np.all(clrd.valuation == clrd2.valuation) + + def test_json_for_val(self, raa: Triangle) -> None: + """ + test JSON round trip for val Triangle + + Parameters + ---------- + raa: Triangle + The CLRD sample data set fixture + + Returns + ------- + None + """ + x = raa.dev_to_val().to_json() + assert cl.read_json(x).equals(raa.dev_to_val()) + + def test_estimator_json_io(self) -> None: + """ + test JSON round trip for Development estimator + + Returns + ------- + None + """ + assert ( + cl.read_json(cl.Development().to_json()).get_params() + == cl.Development().get_params() + ) + + def test_pipeline_json_io(self) -> None: + """ + test JSON round trip for an estimator pipeline + + Returns + ------- + None + """ + pipe = cl.Pipeline( + steps=[("dev", cl.Development()), ("model", cl.BornhuetterFerguson())] + ) + pipe2 = cl.read_json(pipe.to_json()) + assert { + item[0]: item[1].get_params() for item in pipe.get_params()["steps"] + } == {item[0]: item[1].get_params() for item in pipe2.get_params()["steps"]} + + def test_json_subtri(self, raa: Triangle, atol: float) -> None: + """ + test JSON round trip preserves sub-Triangles + + Parameters + ---------- + raa: Triangle + The CLRD sample data set fixture + + atol: float + the absolute tolerance for this test + + Returns + ------- + None + """ + a = cl.read_json(cl.Chainladder().fit_predict(raa).to_json()).full_triangle_ + b = cl.Chainladder().fit_predict(raa).full_triangle_ + # since a and b are meant to be identical, (a - b) should be 0 everywhere + # 0's are turned to NaN's in Triangle arithmetic, resulting in an All-NaN + # warning when aggregating + with pytest.warns(RuntimeWarning, match="All-NaN"): + assert abs(a - b).max().max() < atol + + def test_json_df(self, atol: float) -> None: + """ + test JSON round trip preserves sub-Triangles from Munich Adjustment + + Parameters + ---------- + atol: float + the absolute tolerance for this test + + Returns + ------- + None + """ + x = cl.MunichAdjustment(paid_to_incurred=("paid", "incurred")).fit_transform( + cl.load_sample("mcl") + ) + assert abs(cl.read_json(x.to_json()).lambda_ - x.lambda_).sum() < atol + + +class TestPickle: + """Test JSON IO""" + + def test_to_pickle_read_pickle(self, raa: Triangle) -> None: + """ + test JSON round trip for val Triangle + + Parameters + ---------- + raa: Triangle + The CLRD sample data set fixture + + Returns + ------- + None + """ + import os + import tempfile + + dev = cl.Development(average="simple", n_periods=4).fit(raa) + fd, path = tempfile.mkstemp(suffix=".pkl") + os.close(fd) + try: + dev.to_pickle(path) + restored = cl.read_pickle(path) + assert restored.average == dev.average + assert restored.n_periods == dev.n_periods + np.testing.assert_array_almost_equal(restored.ldf_.values, dev.ldf_.values) + finally: + os.remove(path) + + def test_read_pickle_triangle(self, raa: Triangle, tmp_path: Path) -> None: + """ + Create a triangle, dump a pickle of it, and then read it back in. The ingested + pickle should result in an equal copy of the triangle that was dumped. + + Parameters + ---------- + raa: Triangle + The raa sample data set. + tmp_path: Path + The builtin pytest tmp_path fixture, provides a temporary path to dump the + pickle to. + + Returns + ------- + None + + """ + pkl_path = tmp_path / "triangle.pkl" + with open(pkl_path, "wb") as f: + dill.dump(raa, f) + assert cl.read_pickle(str(pkl_path)).equals(raa) + + def test_triangle_to_pickle( + self, raa: Triangle, clrd: Triangle, tmp_path: Path + ) -> None: + """ + Dump a pickle of a triangle and read it back in. The read-in triangle should + equal the one that was dumped. + + Parameters + ---------- + raa: Triangle + The raa sample data set Triangle. + clrd: Triangle + The clrd sample data set Triangle. + tmp_path: Path + The builtin pytest tmp_path fixture, provides a temporary path to dump the + pickle to. + + Returns + ------- + None + + """ + # Single-dimension case. + raa_path = tmp_path / "raa.pkl" + raa.to_pickle(str(raa_path)) + assert raa_path.is_file() + assert cl.read_pickle(str(raa_path)).equals(raa) + + # Multidimensional case. + clrd_path = tmp_path / "clrd.pkl" + clrd.to_pickle(str(clrd_path)) + assert clrd_path.is_file() + assert cl.read_pickle(str(clrd_path)).equals(clrd) + + def test_read_pickle_estimator(self, raa: Triangle, tmp_path: Path) -> None: + """ + Create an estimator, dump a pickle of it, and then read it back in. The ingested + pickle should result produce the same LDFs that the original estimator does. + + Parameters + ---------- + raa: Triangle + The raa sample data set. + tmp_path: Path + The builtin pytest tmp_path fixture, provides a temporary path to dump the + pickle to. + + Returns + ------- + None + + """ + + pkl_path = tmp_path / "estimator.pkl" + dev = cl.Development().fit(raa) + with open(pkl_path, "wb") as f: + dill.dump(dev, f) + assert dev.ldf_.equals(cl.read_pickle(str(pkl_path)).ldf_) + + def test_pickle_backward_compatibility(self, raa: Triangle) -> None: + """ + Tests backward compatibility with pickles from older versions + + Parameters + ---------- + raa: Triangle + The raa sample data set fixture + + Returns + ------- + None + """ + data_dir: Path = Path(__file__).parent.parent.parent / "utils" / "data" + raa_pickles = list(data_dir.glob("raa*.pkl")) + + for raa_pickle in raa_pickles: + test = cl.read_pickle(raa_pickle) + assert test.equals(raa) + assert repr(test) == repr(raa) + assert ( + cl.Development().fit(test).ldf_.equals(cl.Development().fit(raa).ldf_) + ) diff --git a/chainladder/core/triangle.py b/chainladder/core/triangle.py index bb76492a5..32f3f0463 100644 --- a/chainladder/core/triangle.py +++ b/chainladder/core/triangle.py @@ -7,24 +7,24 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. from __future__ import annotations -import pandas as pd -import numpy as np import warnings + +import numpy as np +import pandas as pd + +from chainladder import __dt64_dtype__, _warn_dask_parallel_deprecated, options +from chainladder._config.deprecation import _deprecated_rename_argument +from chainladder.core.axis import TriangleAxis, _set_columns, _set_index from chainladder.core.base import TriangleBase -from chainladder.core.axis import TriangleAxis -from chainladder.core.axis import _set_columns -from chainladder.core.axis import _set_index -from chainladder.utils.sparse import sp -from chainladder.core.slice import VirtualColumns from chainladder.core.correlation import DevelopmentCorrelation, ValuationCorrelation +from chainladder.core.slice import VirtualColumns +from chainladder.utils.sparse import sp from chainladder.utils.utility_functions import ( concat, num_to_nan, num_to_value, to_period, ) -from chainladder import options, _warn_dask_parallel_deprecated, __dt64_dtype__ -from chainladder._config.deprecation import _deprecated_rename_argument try: import dask.bag as db @@ -32,22 +32,23 @@ db = None from typing import ( + TYPE_CHECKING, Any, - cast, Literal, Optional, - TYPE_CHECKING, Self, + cast, ) if TYPE_CHECKING: - from pandas import DataFrame, Series - from chainladder.core.style import Styler - from chainladder.core.typing import BackendArray from numpy.typing import ArrayLike + from pandas import DataFrame, Series from pandas._libs.tslibs.timestamps import Timestamp # noqa from pandas.core.interchange.dataframe_protocol import DataFrame as DataFrameXchg + from chainladder.core.style import Styler + from chainladder.core.typing import BackendArray + class Triangle(TriangleBase): """ @@ -135,11 +136,13 @@ class Triangle(TriangleBase): development_grain: str The grain of the development vector ('Y', 'S', 'Q', 'M') shape: tuple - The 4D shape of the triangle instance with axes corresponding to (index, columns, origin, development) + The 4D shape of the triangle instance with axes corresponding to + (index, columns, origin, development) link_ratio, age_to_age: Triangle Displays age-to-age ratios for the triangle. disposal_rate_tri: Triangle - Displays actual disposal rates by origin and development; must have ``ultimate_`` + Displays actual disposal rates by origin and development; must have + ``ultimate_`` valuation_date : date The latest valuation date of the data latest_diagonal: Triangle @@ -671,8 +674,9 @@ def __init__( if cumulative is None: warnings.warn( - "The cumulative property of your triangle is not set. This may result in " - "undesirable behavior. In a future release this will result in an error." + "The cumulative property of your triangle is not set. " + "This may result in undesirable behavior. In a future " + "release this will result in an error." ) self.is_cumulative: bool = cumulative @@ -1150,7 +1154,7 @@ def is_pattern(self) -> bool: True """ - return self._pattern + return getattr(self, "_pattern", False) @is_pattern.setter def is_pattern(self, pattern: bool): @@ -1159,7 +1163,8 @@ def is_pattern(self, pattern: bool): @property def is_disposal_rate(self) -> bool: """ - Indicates whether the Triangle holds disposal rates (additive ratios going from head to tail) + Indicates whether the Triangle holds disposal rates (additive ratios + going from head to tail) """ if hasattr(self, "_is_disposal_rate"): return self._is_disposal_rate @@ -1173,7 +1178,8 @@ def align_pattern( self, X: Triangle, sample_weight: Triangle | None = None ) -> Triangle: """ - Vertically align a selected pattern to origin period latest diagonal. Triangle must be a selected pattern. + Vertically align a selected pattern to origin period latest diagonal. + Triangle must be a selected pattern. Parameters ---------- @@ -1189,7 +1195,7 @@ def align_pattern( Triangle of selected pattern across origin periods """ - if not self._pattern: + if not self.is_pattern: raise ValueError( "Triangle is not a selected pattern, such as .ldf_ or .cdf_" ) @@ -1308,9 +1314,10 @@ def latest_diagonal(self) -> Triangle: @property def link_ratio(self) -> Triangle: """ - Displays age-to-age ratios for the triangle. If the calling Triangle object already has the - self.is_pattern set to true (i.e., it is already a set of link ratios or development patterns), - this property simply returns itself. + Displays age-to-age ratios for the triangle. If the calling Triangle + object already has the self.is_pattern set to true (i.e., it is already + a set of link ratios or development patterns), this property simply + returns itself. Returns ------- diff --git a/chainladder/utils/data/raa_0_10_0.pkl b/chainladder/utils/data/raa_0_10_0.pkl new file mode 100644 index 000000000..a355ec318 Binary files /dev/null and b/chainladder/utils/data/raa_0_10_0.pkl differ diff --git a/chainladder/utils/data/raa_0_8_24.pkl b/chainladder/utils/data/raa_0_8_24.pkl new file mode 100644 index 000000000..f400a0b32 Binary files /dev/null and b/chainladder/utils/data/raa_0_8_24.pkl differ diff --git a/chainladder/utils/data/raa_0_9_1.pkl b/chainladder/utils/data/raa_0_9_1.pkl new file mode 100644 index 000000000..56118f3cd Binary files /dev/null and b/chainladder/utils/data/raa_0_9_1.pkl differ diff --git a/chainladder/utils/tests/test_utilities.py b/chainladder/utils/tests/test_utilities.py index 041073152..dda1b3d70 100644 --- a/chainladder/utils/tests/test_utilities.py +++ b/chainladder/utils/tests/test_utilities.py @@ -6,7 +6,6 @@ from pathlib import Path from typing import TYPE_CHECKING -import dill import numpy as np import pandas as pd import pytest @@ -214,23 +213,6 @@ def test_concat_immutability(raa): assert u.equals(u_new) -def test_to_pickle_read_pickle(raa): - import os - import tempfile - - dev = cl.Development(average="simple", n_periods=4).fit(raa) - fd, path = tempfile.mkstemp(suffix=".pkl") - os.close(fd) - try: - dev.to_pickle(path) - restored = cl.read_pickle(path) - assert restored.average == dev.average - assert restored.n_periods == dev.n_periods - np.testing.assert_array_almost_equal(restored.ldf_.values, dev.ldf_.values) - finally: - os.remove(path) - - def test_maximum_minimum_1(raa): ult_vol = ( cl @@ -409,7 +391,8 @@ def test_load_sample_clrd2025() -> None: def test_date_delta_adjustment() -> None: """ - Tests the date adjustment depending on Pandas default precision, nanosecond for Pandas 2, microsecond for Pandas 3. + Tests the date adjustment depending on Pandas default precision, + nanosecond for Pandas 2, microsecond for Pandas 3. """ result = date_delta_adjustment("2025-11-01") @@ -421,90 +404,10 @@ def test_date_delta_adjustment() -> None: assert result == expected -def test_read_pickle_triangle(raa: Triangle, tmp_path: Path) -> None: - """ - Create a triangle, dump a pickle of it, and then read it back in. The ingested pickle should result - in an equal copy of the triangle that was dumped. - - Parameters - ---------- - raa: Triangle - The raa sample data set. - tmp_path: Path - The builtin pytest tmp_path fixture, provides a temporary path to dump the pickle to. - - Returns - ------- - None - - """ - pkl_path = tmp_path / "triangle.pkl" - with open(pkl_path, "wb") as f: - dill.dump(raa, f) - assert cl.read_pickle(str(pkl_path)).equals(raa) - - -def test_triangle_to_pickle(raa: Triangle, clrd: Triangle, tmp_path: Path) -> None: - """ - Dump a pickle of a triangle and read it back in. The read-in triangle should - equal the one that was dumped. - - Parameters - ---------- - raa: Triangle - The raa sample data set Triangle. - clrd: Triangle - The clrd sample data set Triangle. - tmp_path: Path - The builtin pytest tmp_path fixture, provides a temporary path to dump the pickle to. - - Returns - ------- - None - - """ - # Single-dimension case. - raa_path = tmp_path / "raa.pkl" - raa.to_pickle(str(raa_path)) - assert raa_path.is_file() - assert cl.read_pickle(str(raa_path)).equals(raa) - - # Multidimensional case. - clrd_path = tmp_path / "clrd.pkl" - clrd.to_pickle(str(clrd_path)) - assert clrd_path.is_file() - assert cl.read_pickle(str(clrd_path)).equals(clrd) - - -def test_read_pickle_estimator(raa: Triangle, tmp_path: Path) -> None: - """ - Create an estimator, dump a pickle of it, and then read it back in. The ingested pickle should result - produce the same LDFs that the original estimator does. - - Parameters - ---------- - raa: Triangle - The raa sample data set. - tmp_path: Path - The builtin pytest tmp_path fixture, provides a temporary path to dump the pickle to. - - Returns - ------- - None - - """ - - pkl_path = tmp_path / "estimator.pkl" - dev = cl.Development().fit(raa) - with open(pkl_path, "wb") as f: - dill.dump(dev, f) - assert dev.ldf_.equals(cl.read_pickle(str(pkl_path)).ldf_) - - def test_concat_type_error() -> None: """ - Supply a string to cl.concat() and raise a TypeError. Concat expects a list or tuple, so supplying a string - will raise the error. + Supply a string to cl.concat() and raise a TypeError. Concat expects + a list or tuple, so supplying a string will raise the error. Returns ------- @@ -517,9 +420,12 @@ def test_concat_type_error() -> None: def test_concat_empty_error() -> None: """ - Supply an empty list to cl.concat(). Trigger a ValueError since the length of the list must be greater than 0. + Supply an empty list to cl.concat(). Trigger a ValueError since the + length of the list must be greater than 0. + Returns ------- + None """ with pytest.raises(ValueError): @@ -528,8 +434,9 @@ def test_concat_empty_error() -> None: def test_concat_mismatched_columns(clrd: Triangle) -> None: """ - Concatenate two triangles where each has a column that the other does not have. This creates new columns - for each triangle which then get filled with xp.nan. + Concatenate two triangles where each has a column that the other does + not have. This creates new columns for each triangle which then get + filled with xp.nan. Parameters ---------- @@ -558,8 +465,8 @@ def test_concat_mismatched_columns(clrd: Triangle) -> None: def test_concat_sort(clrd: Triangle) -> None: """ - Concat two triangles with indexes in reverse alphabetical order then sort the index. Check to see - that the index gets sorted alphabetically. + Concat two triangles with indexes in reverse alphabetical order then + sort the index. Check to see that the index gets sorted alphabetically. Parameters ---------- @@ -599,8 +506,8 @@ def test_concat_axis1(raa: Triangle) -> None: def test_concat_axis1_duplicate_columns(raa: Triangle) -> None: """ - Concat two triangles along the column axis with the same column names. Raise an assertion error since - columns must be unique. + Concat two triangles along the column axis with the same column names. + Raise an assertion error since columns must be unique. Parameters ---------- @@ -640,7 +547,10 @@ def test_concat_ignore_index_axes(raa: Triangle) -> None: def test_concat_axis_1_index_dtype_relaxed(raa: Triangle) -> None: - """Verify concat on axis 1 succeeds when index values match but row index labels differ.""" + """ + Verify concat on axis 1 succeeds when index values match but + row index labels differ. + """ t1 = copy.deepcopy(raa).rename("columns", ["A"]) t2 = copy.deepcopy(raa).rename("columns", ["B"]) t2.index = pd.DataFrame(t2.index.values, columns=t2.index.columns, index=[10]) @@ -661,7 +571,8 @@ def test_concat_axis_1_index_nan_preserved(raa: Triangle) -> None: def test_maximum_2(raa: Triangle) -> None: """ - Run cl.maximum(raa, 5000) and check if each element in the resulting triangle is at least 5000. + Run cl.maximum(raa, 5000) and check if each element in the resulting + triangle is at least 5000. Parameters ---------- @@ -680,7 +591,8 @@ def test_maximum_2(raa: Triangle) -> None: def test_minimum_2(raa): """ - Run cl.minimum(raa, 5000) and check if each element in the resulting triangle is at most 5000. + Run cl.minimum(raa, 5000) and check if each element in the resulting + triangle is at most 5000. Parameters ---------- @@ -699,7 +611,8 @@ def test_minimum_2(raa): def test_reset_option() -> None: """ - Change some of the options and then reset them. Values after reset should match the original values. + Change some of the options and then reset them. Values after reset should + match the original values. Returns ------- @@ -731,7 +644,8 @@ def test_reset_option() -> None: def test_options_defaults() -> None: """ - When initialized, default options should be correct and accessible from the options variable. + When initialized, default options should be correct and accessible from + the options variable. Returns ------- @@ -762,7 +676,8 @@ def test_get_option() -> None: def test_set_option_consistency() -> None: """ - When set_option changes an option value, get_option should return the new option value. + When set_option changes an option value, get_option should return the + new option value. Returns ------- @@ -1146,8 +1061,8 @@ def test_dask_parallel_numpy_groupby_no_warning( def test_describe_option(capsys: CaptureFixture[str]) -> None: """ - Supply an option to cl.options.describe_option(). Attribute name, type, default/current - settings should be captured in the output. + Supply an option to cl.options.describe_option(). Attribute name, type, + default/current settings should be captured in the output. Parameters ---------- @@ -1168,8 +1083,8 @@ def test_describe_option(capsys: CaptureFixture[str]) -> None: def test_describe_option_multi(capsys) -> None: """ - Supply two options to cl.options.describe_option(). Attribute names, types, default/current - settings should be captured in the output. + Supply two options to cl.options.describe_option(). Attribute names, types, + default/current settings should be captured in the output. Parameters ---------- @@ -1215,8 +1130,8 @@ def test_describe_option_all(capsys) -> None: def test_describe_option_return_string() -> None: """ - Execute cl.options.desribe_option() with _print_desc=False. Should return a string. Check - if attribute info is in the string. + Execute cl.options.desribe_option() with _print_desc=False. Should return a + string. Check if attribute info is in the string. Returns ------- @@ -1267,8 +1182,8 @@ def test_get_option_missing_pat_raises() -> None: def test_describe_option_no_docstring_match(monkeypatch: MonkeyPatch) -> None: """ - When the class docstring has no entry for an option, describe_option should fall back - to 'No description available.' rather than raising an error. + When the class docstring has no entry for an option, describe_option should + fall back to 'No description available.' rather than raising an error. Parameters ---------- @@ -1286,7 +1201,8 @@ def test_describe_option_no_docstring_match(monkeypatch: MonkeyPatch) -> None: def test_describe_option_invalid() -> None: """ - Execute cl.options.desribe_option() with an invalid argument. Should raise a ValueError. + Execute cl.options.desribe_option() with an invalid argument. Should raise + a ValueError. Returns ------- @@ -1299,7 +1215,8 @@ def test_describe_option_invalid() -> None: def test_both_pat_and_option_raises() -> None: """ - Passing both pat and option to get_option, set_option, or reset_option should raise TypeError. + Passing both pat and option to get_option, set_option, or reset_option should + raise TypeError. """ with pytest.raises(TypeError, match="Cannot specify both"): cl.options.get_option(pat="ARRAY_BACKEND", option="ARRAY_BACKEND") diff --git a/chainladder/utils/utility_functions.py b/chainladder/utils/utility_functions.py index e581face7..566ba67dc 100644 --- a/chainladder/utils/utility_functions.py +++ b/chainladder/utils/utility_functions.py @@ -253,7 +253,10 @@ def read_pickle(path): """ with open(path, "rb") as pkl: - return dill.load(pkl) + obj = dill.load(pkl) + if "is_pattern" in obj.__dict__ and "_pattern" not in obj.__dict__: + obj.__dict__["_pattern"] = obj.__dict__.pop("is_pattern") + return obj @_deprecated_rename_argument("development", "valuation", remove_in_version="v2.0")