diff --git a/CHANGELOG.md b/CHANGELOG.md index d2c238a094..cff5ce73b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Code freeze date: YYYY-MM-DD - Adds `MeasureConfig` and related dataclasses for new `Measure` object retrocompatibility and (de)serialization capabilities [#1276](https://github.com/CLIMADA-project/climada_python/pull/1276) - Add inter- and extrapolation options to `ImpactFreqCurve` with method `interpolate` [#1252](https://github.com/CLIMADA-project/climada_python/pull/1252) - Add `trajectory` module to ease the handling of multiple risk assessment at different point in time, and enable interpolation in between. [#1197](https://github.com/CLIMADA-project/climada_python/pull/1197), [#1198](https://github.com/CLIMADA-project/climada_python/pull/1198), [#1199](https://github.com/CLIMADA-project/climada_python/pull/1199), [#1200](https://github.com/CLIMADA-project/climada_python/pull/1197), [#1201](https://github.com/CLIMADA-project/climada_python/pull/1197) +- `climada.util.dataframe_handling.read_excel` a wrapper around `pandas.read_excel` that suppresses unrelated user-warnings [#1335](https://github.com/CLIMADA-project/climada_python/issues/1335) ### Changed diff --git a/climada/engine/impact.py b/climada/engine/impact.py index a4755c42e7..098b923eff 100644 --- a/climada/engine/impact.py +++ b/climada/engine/impact.py @@ -26,21 +26,21 @@ import datetime as dt import logging import warnings -from collections.abc import Collection +from collections.abc import Collection, Iterable from dataclasses import dataclass, field from itertools import zip_longest from pathlib import Path -from typing import Any, Iterable, Union +from typing import Any, Union import contextily as ctx import geopandas as gpd import h5py -import matplotlib.animation as animation import matplotlib.pyplot as plt import numpy as np import pandas as pd import xlsxwriter from deprecation import deprecated +from matplotlib import animation from matplotlib.colors import Normalize from pandas.api.types import is_string_dtype from pyproj import CRS as pyprojCRS @@ -49,6 +49,7 @@ from tqdm import tqdm import climada.util.coordinates as u_coord +import climada.util.dataframe_handling as u_pd import climada.util.dates_times as u_dt import climada.util.interpolation as u_interp import climada.util.plot as u_plot @@ -1567,7 +1568,7 @@ def from_excel(cls, file_name): Impact from excel file """ LOGGER.info("Reading %s", file_name) - imp_df = pd.read_excel(file_name) + imp_df = u_pd.read_excel(file_name) imp = cls(haz_type=str(imp_df["haz_type"][0])) imp.unit = imp_df["unit"][0] diff --git a/climada/entity/_legacy_measures/measure_set.py b/climada/entity/_legacy_measures/measure_set.py index 228788ba15..5aea38db3e 100755 --- a/climada/entity/_legacy_measures/measure_set.py +++ b/climada/entity/_legacy_measures/measure_set.py @@ -24,13 +24,12 @@ import ast import copy import logging -from typing import List, Optional import numpy as np -import pandas as pd import xlsxwriter from matplotlib import colormaps as cm +import climada.util.dataframe_handling as u_pd import climada.util.hdf5_handler as u_hdf5 from .base import Measure @@ -99,7 +98,7 @@ class MeasureSet: Use the available methods instead. """ - def __init__(self, measure_list: Optional[List[Measure]] = None): + def __init__(self, measure_list: list[Measure] | None = None): """Initialize a new MeasureSet object with specified data. Parameters @@ -130,7 +129,7 @@ def __init__(self, measure_list: Optional[List[Measure]] = None): for meas in measure_list: self.append(meas) - def clear(self, _data: Optional[dict] = None): + def clear(self, _data: dict | None = None): """Reinitialize attributes. Parameters @@ -544,7 +543,7 @@ def read_att_excel(measures, dfr, var_names): measures.append(Measure(**meas_kwargs)) - dfr = pd.read_excel(file_name, var_names["sheet_name"]) + dfr = u_pd.read_excel(file_name, var_names["sheet_name"]) dfr = dfr.fillna("") meas_set = cls() try: diff --git a/climada/entity/disc_rates/base.py b/climada/entity/disc_rates/base.py index e18daaf919..626dd51df6 100755 --- a/climada/entity/disc_rates/base.py +++ b/climada/entity/disc_rates/base.py @@ -32,6 +32,7 @@ import xlsxwriter import climada.util.checker as u_check +import climada.util.dataframe_handling as u_pd import climada.util.finance as u_fin import climada.util.hdf5_handler as u_hdf5 @@ -293,7 +294,7 @@ def from_excel(cls, file_name, var_names=None): """ if var_names is None: var_names = DEF_VAR_EXCEL - dfr = pd.read_excel(file_name, var_names["sheet_name"]) + dfr = u_pd.read_excel(file_name, var_names["sheet_name"]) try: years = dfr[var_names["col_name"]["year"]].values.astype(int, copy=False) rates = dfr[var_names["col_name"]["disc"]].values diff --git a/climada/entity/entity_def.py b/climada/entity/entity_def.py index c1bc3b2550..6018a2a474 100755 --- a/climada/entity/entity_def.py +++ b/climada/entity/entity_def.py @@ -24,9 +24,8 @@ import logging from typing import Optional -import pandas as pd - -from climada.entity._legacy_measures.measure_set import Measure, MeasureSet +import climada.util.dataframe_handling as u_pd +from climada.entity._legacy_measures.measure_set import MeasureSet from climada.entity.disc_rates.base import DiscRates from climada.entity.exposures.base import Exposures from climada.entity.impact_funcs.impact_func_set import ImpactFuncSet @@ -128,7 +127,7 @@ def from_excel(cls, file_name): The entity from excel file """ - exp = Exposures(pd.read_excel(file_name)) + exp = Exposures(u_pd.read_excel(file_name)) disc_rates = DiscRates.from_excel(file_name) impf_set = ImpactFuncSet.from_excel(file_name) diff --git a/climada/entity/exposures/litpop/litpop.py b/climada/entity/exposures/litpop/litpop.py index ef28b3ba5d..c7f6a5d5b6 100644 --- a/climada/entity/exposures/litpop/litpop.py +++ b/climada/entity/exposures/litpop/litpop.py @@ -31,6 +31,7 @@ from shapefile import Shape import climada.util.coordinates as u_coord +import climada.util.dataframe_handling as u_pd import climada.util.finance as u_fin from climada import CONFIG from climada.entity.exposures.base import DEF_REF_YEAR, INDICATOR_IMPF, Exposures @@ -1493,7 +1494,7 @@ def _grp_read(country_iso3, admin1_info=None, data_dir=SYSTEM_DIR): ) if file_name is not None: # open spreadsheet and identify relevant columns: - admin1_xls_data = pd.read_excel(file_name) + admin1_xls_data = u_pd.read_excel(file_name) if admin1_xls_data.get("State_Province") is None: admin1_xls_data = admin1_xls_data.rename( columns={admin1_xls_data.columns[0]: "State_Province"} diff --git a/climada/entity/exposures/test/test_base.py b/climada/entity/exposures/test/test_base.py index 1ac9062ecf..a5c5f35749 100644 --- a/climada/entity/exposures/test/test_base.py +++ b/climada/entity/exposures/test/test_base.py @@ -31,6 +31,7 @@ from sklearn.metrics import DistanceMetric import climada.util.coordinates as u_coord +import climada.util.dataframe_handling as u_pd from climada import CONFIG from climada.entity import LitPop from climada.entity.exposures.base import ( @@ -425,7 +426,7 @@ class TestIO(unittest.TestCase): def test_read_template_pass(self): """Wrong exposures definition""" - df = pd.read_excel(ENT_TEMPLATE_XLS) + df = u_pd.read_excel(ENT_TEMPLATE_XLS) exp_df = Exposures(df) # set metadata exp_df.ref_year = 2020 @@ -434,7 +435,7 @@ def test_read_template_pass(self): def test_handling_unit_conflicts_pass(self): """Check that the value_unit is correctly set when there are conflicting value_unit definitions in the data frame and the meta attribute.""" - df = pd.read_excel(ENT_TEMPLATE_XLS) + df = u_pd.read_excel(ENT_TEMPLATE_XLS) exp_df = Exposures(df, meta={"value_unit": "XSD"}, value_unit="XSD") exp_df.check() self.assertEqual(exp_df.value_unit, "XSD") @@ -444,7 +445,7 @@ def test_handling_unit_conflicts_pass(self): def test_io_hdf5_pass(self): """write and read hdf5""" exp = Exposures( - pd.read_excel(ENT_TEMPLATE_XLS), crs="epsg:32632", ref_year=2020 + u_pd.read_excel(ENT_TEMPLATE_XLS), crs="epsg:32632", ref_year=2020 ) # add another geometry column diff --git a/climada/entity/impact_funcs/impact_func_set.py b/climada/entity/impact_funcs/impact_func_set.py index b6f4cf73d7..bf3200b158 100755 --- a/climada/entity/impact_funcs/impact_func_set.py +++ b/climada/entity/impact_funcs/impact_func_set.py @@ -23,14 +23,15 @@ import copy import logging +from collections.abc import Iterable from itertools import repeat -from typing import Iterable, Optional, Union, overload +from typing import Optional, Union, overload import matplotlib.pyplot as plt import numpy as np -import pandas as pd import xlsxwriter +import climada.util.dataframe_handling as u_pd import climada.util.hdf5_handler as u_hdf5 import climada.util.plot as u_plot from climada.entity.impact_funcs.base import ImpactFunc @@ -424,7 +425,7 @@ def from_excel(cls, file_name, var_names=None): """ if var_names is None: var_names = DEF_VAR_EXCEL - dfr = pd.read_excel(file_name, var_names["sheet_name"]) + dfr = u_pd.read_excel(file_name, var_names["sheet_name"]) imp_func_set = cls() imp_func_set._fill_dfr(dfr, var_names) diff --git a/climada/hazard/centroids/centr.py b/climada/hazard/centroids/centr.py index 7fcc769b52..4b4c4b9bff 100644 --- a/climada/hazard/centroids/centr.py +++ b/climada/hazard/centroids/centr.py @@ -39,6 +39,7 @@ from shapely.geometry.point import Point import climada.util.coordinates as u_coord +import climada.util.dataframe_handling as u_pd from climada.util.constants import DEF_CRS __all__ = ["Centroids"] @@ -882,7 +883,7 @@ def from_excel(cls, file_path, sheet_name=None): """ if sheet_name is None: sheet_name = "centroids" - df = pd.read_excel(file_path, sheet_name) + df = u_pd.read_excel(file_path, sheet_name) return cls._from_dataframe(df) def write_excel(self, file_path): @@ -1053,7 +1054,7 @@ def _gdf_from_legacy_hdf5(data): def _legacy_from_excel(cls, file_name, var_names): LOGGER.info("Reading %s", file_name) try: - df = pd.read_excel(file_name, var_names["sheet_name"]) + df = u_pd.read_excel(file_name, var_names["sheet_name"]) df = df.rename(columns=var_names["col_name"]) except KeyError as err: raise KeyError("Not existing variable: %s" % str(err)) from err diff --git a/climada/hazard/io.py b/climada/hazard/io.py index 03e608d050..ba40f6e42f 100644 --- a/climada/hazard/io.py +++ b/climada/hazard/io.py @@ -25,11 +25,10 @@ import pathlib import warnings from collections.abc import Collection -from typing import Any, Dict, Optional, Union +from typing import Any, Union import h5py import numpy as np -import pandas as pd import rasterio import xarray as xr from deprecation import deprecated @@ -37,6 +36,7 @@ import climada.util.constants as u_const import climada.util.coordinates as u_coord +import climada.util.dataframe_handling as u_pd import climada.util.hdf5_handler as u_hdf5 from climada.hazard.centroids.centr import Centroids @@ -317,8 +317,8 @@ def from_xarray_raster( intensity_unit: str, *, intensity: str = "intensity", - coordinate_vars: Optional[Dict[str, str]] = None, - data_vars: Optional[Dict[str, str]] = None, + coordinate_vars: dict[str, str] | None = None, + data_vars: dict[str, str] | None = None, crs: str = u_const.DEF_CRS, rechunk: bool = False, open_dataset_kws: dict[str, Any] | None = None, @@ -604,7 +604,7 @@ def from_xarray_raster( return cls(**cls._check_and_cast_attrs(kwargs)) @staticmethod - def _check_and_cast_attrs(attrs: Dict[str, Any]) -> Dict[str, Any]: + def _check_and_cast_attrs(attrs: dict[str, Any]) -> dict[str, Any]: """Check the validity of the hazard attributes given and cast to correct type if required and possible. @@ -731,7 +731,7 @@ def _check_and_cast_elements( return attrs @staticmethod - def _attrs_to_kwargs(attrs: Dict[str, Any], num_events: int) -> Dict[str, Any]: + def _attrs_to_kwargs(attrs: dict[str, Any], num_events: int) -> dict[str, Any]: """Transform attributes to init kwargs or use default values If attributes are missing from ``attrs``, this method will use a sensible default @@ -1092,7 +1092,7 @@ def _read_att_mat(data, file_name, var_names, centroids): @staticmethod def _read_att_excel(file_name, var_names, centroids): """Read Excel hazard's attributes.""" - dfr = pd.read_excel(file_name, var_names["sheet_name"]["freq"]) + dfr = u_pd.read_excel(file_name, var_names["sheet_name"]["freq"]) num_events = dfr.shape[0] attrs = dict() @@ -1106,7 +1106,7 @@ def _read_att_excel(file_name, var_names, centroids): ) attrs["event_name"] = dfr[var_names["col_name"]["even_name"]].values.tolist() - dfr = pd.read_excel(file_name, var_names["sheet_name"]["inten"]) + dfr = u_pd.read_excel(file_name, var_names["sheet_name"]["inten"]) # number of events (ignore centroid_ID column) # check the number of events is the same as the one in the frequency if dfr.shape[1] - 1 is not num_events: diff --git a/climada/hazard/test/test_tc_tracks_synth.py b/climada/hazard/test/test_tc_tracks_synth.py index 9ee889df3b..017efd0945 100644 --- a/climada/hazard/test/test_tc_tracks_synth.py +++ b/climada/hazard/test/test_tc_tracks_synth.py @@ -434,6 +434,7 @@ def test_wrong_decay_pass(self): 4: (1.0714354641894077, 0.002783787561718677), } track_gen.attrs["orig_event_flag"] = False + # the reference central pressure may need to be updated when IBTrACS change their files, # which may happen even without an apparent version change cp_ref = np.array([1013.0, 1012.0] + [1011.0] * 6 + [1010.0] * 10) diff --git a/climada/test/test_plot.py b/climada/test/test_plot.py index 89adaf0e5c..e9e443f476 100644 --- a/climada/test/test_plot.py +++ b/climada/test/test_plot.py @@ -21,14 +21,13 @@ import copy import unittest -import urllib from pathlib import Path import contextily as ctx import matplotlib.pyplot as plt import numpy as np -import pandas as pd +import climada.util.dataframe_handling as u_pd from climada.engine import CostBenefit, ImpactCalc, ImpactFreqCurve from climada.engine.unsequa import UncOutput from climada.entity import ( @@ -40,7 +39,7 @@ Measure, MeasureSet, ) -from climada.hazard import Centroids, Hazard +from climada.hazard import Hazard from climada.test import get_test_file from climada.util.api_client import Client from climada.util.constants import ENT_DEMO_TODAY, HAZ_DEMO_FL, TEST_UNC_OUTPUT_COSTBEN @@ -117,7 +116,7 @@ def test_hazard_rp_intensity(self): def test_exposures_value_pass(self): """Plot exposures values.""" - myexp = pd.read_excel(ENT_DEMO_TODAY) + myexp = u_pd.read_excel(ENT_DEMO_TODAY) myexp = Exposures(myexp) myexp.check() myexp.description = "demo_today" diff --git a/climada/util/dataframe_handling.py b/climada/util/dataframe_handling.py index 6a5c37a95a..7bbe919bb2 100644 --- a/climada/util/dataframe_handling.py +++ b/climada/util/dataframe_handling.py @@ -19,6 +19,8 @@ Define functions to handle with coordinates """ +import warnings + import pandas as pd @@ -63,3 +65,30 @@ def reorder_dataframe_columns( new_column_order.extend(remaining_columns) return dataframe[new_column_order] + + +def read_excel(*args, **kwargs): + """A wrapper around pandas.read_excel that suppresses the openpyxl warning + 'Unknown extension is not supported and will be removed', from openpyxl.worksheet._read_only + + Parameters + ---------- + *args + Positional arguments forwarded to :func:`pandas.read_excel`. The first + argument is typically the path to an Excel file or a file-like object. + **kwargs + Keyword arguments forwarded unchanged to :func:`pandas.read_excel`, + such as ``sheet_name``, ``header``, and ``usecols``. + + Returns + ------- + pandas.DataFrame + """ + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="Unknown extension is not supported and will be removed", + category=UserWarning, + module="openpyxl.worksheet._reader", + ) + return pd.read_excel(*args, **kwargs) diff --git a/climada/util/finance.py b/climada/util/finance.py index 7ae54fcc45..27f5edaa85 100644 --- a/climada/util/finance.py +++ b/climada/util/finance.py @@ -32,6 +32,7 @@ import requests from cartopy.io import shapereader +import climada.util.dataframe_handling as u_pd from climada.util.constants import SYSTEM_DIR from climada.util.files_handler import download_file @@ -289,7 +290,7 @@ def world_bank(cntry_iso, ref_year, info_ind): if not fn_ig.is_file(): file_down = download_file(WORLD_BANK_INC_GRP) shutil.move(file_down, fn_ig) - dfr_wb = pd.read_excel(fn_ig, "Country Analytical History", skiprows=5) + dfr_wb = u_pd.read_excel(fn_ig, "Country Analytical History", skiprows=5) dfr_wb = dfr_wb.drop(dfr_wb.index[0:5]).set_index("Unnamed: 0") dfr_wb = dfr_wb.replace( INCOME_GRP_WB_TABLE.keys(), INCOME_GRP_WB_TABLE.values()