Fix WorldBank urls and outdated data usage

CHANGELOG.md

@@ -21,6 +21,9 @@ Code freeze date: YYYY-MM-DD
 ### Changed
 
 - Updated Impact Calculation Tutorial (`doc.climada_engine_Impact.ipynb`) [#1095](https://github.com/CLIMADA-project/climada_python/pull/1095).
+- `climada.util.finance`: `world_bank_wealth_account` now fetches data via the
+  World Bank Indicators API instead of downloading a ZIP/CSV file.
+  Updated available year range from 1995–2014 to 1995–2020. [#1292](https://github.com/CLIMADA-project/climada_python/pull/1292)
 - Makes current `measure` module a legacy module, moving it to `_legacy_measure`, to retain compatibility with `CostBenefit` class and various tests. [#1274](https://github.com/CLIMADA-project/climada_python/pull/1274)
 - `HazardForecast.quantile` and `ImpactForecast.quantile` compute quantiles block-wise instead of densifying the entire sparse matrix, greatly reducing peak memory. Results are unchanged. [#1203](https://github.com/CLIMADA-project/climada_python/issues/1203)
 
@@ -29,6 +32,8 @@ Code freeze date: YYYY-MM-DD
 - Preserve explicitly mapped dates in `HazardForecast.from_xarray_raster` [#1305](https://github.com/CLIMADA-project/climada_python/issues/1305).
 - Fixed asset count in impact logging message [#1195](https://github.com/CLIMADA-project/climada_python/pull/1195).
 - `Hazard.from_raster_xarray` now returns a sparse matrix instead of a sparse array [#1261](https://github.com/CLIMADA-project/climada_python/pull/1261).
+- `climada.util.finance`: Fixed broken `WORLD_BANK_INC_GRP` download URL
+  (`databank.worldbank.org`) whose SSL certificate had expired. [#1292](https://github.com/CLIMADA-project/climada_python/pull/1292)
 - `ImpactCalc.impact` now raises a clear `ValueError` when the supplied `Hazard` contains no events, instead of failing later inside `np.array_split` with an obscure message [#814](https://github.com/CLIMADA-project/climada_python/issues/814).
 - Fix TCTracks.from_FAST duplicate loading from year loop [#1269](github.com/CLIMADA-project/climada_python/pull/1269)
 
@@ -38,6 +43,8 @@ Code freeze date: YYYY-MM-DD
 ### Removed
 - `climada.util.earth_engine.py` Google Earth Engine methods did not facilitate direct use of GEE data in CLIMADA. Code was relocated to [climada-snippets](https://github.com/CLIMADA-project/climada-snippets). [#1109](https://github.com/CLIMADA-project/climada_python/pull/1109)
 - `doc.climada_util_earth_engine.ipynb` Tutorial about GEE not relevant to CLIMADA Core. Tutorial notebook was relocated to [climada-snippets](https://github.com/CLIMADA-project/climada-snippets). [#1109](https://github.com/CLIMADA-project/climada_python/pull/1109)
+- `climada.util.finance`: Removed constants `WORLD_BANK_WEALTH_ACC` and
+  `FILE_WORLD_BANK_WEALTH_ACC`. [#1292](https://github.com/CLIMADA-project/climada_python/pull/1292)
 - `doc.development.Guide_Euler.ipynb` tutorial is not part of the climada docs anymore. For using climada on the ETHZ HPC Cluster Euler, see https://gitlab.ethz.ch/climada/climada-on-euler/ instead. [#1329](https://github.com/CLIMADA-project/climada_python/pull/1329)
 
 ## 6.1.0

climada/entity/exposures/litpop/litpop.py

@@ -145,7 +145,7 @@ def from_countries(
 
             * 'pc': produced capital (Source: World Bank), incl. manufactured or
               built assets such as machinery, equipment, and physical structures
-              `pc` is in constant 2014 USD.
+              `pc` is in real chained 2019 USD.
             * 'pop': population count (source: GPW, same as gridded population).
               The unit is 'people'.
             * 'gdp': gross-domestic product (Source: World Bank) [USD]
@@ -560,7 +560,7 @@ def from_shape_and_countries(
 
             * 'pc': produced capital (Source: World Bank), incl. manufactured or
               built assets such as machinery, equipment, and physical structures
-              (pc is in constant 2014 USD)
+              (pc is in real chained 2019 USD)
             * 'pop': population count (source: GPW, same as gridded population).
               The unit is 'people'.
             * 'gdp': gross-domestic product (Source: World Bank) [USD]
@@ -1196,11 +1196,11 @@ def _get_total_value_per_country(cntry_iso3a, fin_mode, reference_year):
         to the grid points within the country
             * 'pc': produced capital (Source: World Bank), incl. manufactured or
               built assets such as machinery, equipment, and physical structures
-              (pc is in constant 2014 USD)
+              (pc is in real chained 2019 USD)
             * 'pc_land': produced capital (Source: World Bank), incl. manufactured or
               built assets such as machinery, equipment, physical structures,
               and land value for built-up land.
-              (pc is in constant 2014 USD)
+              (pc is in real chained 2019 USD)
             * 'gdp': gross-domestic product (Source: World Bank) [USD]
             * 'income_group': gdp multiplied by country's income group+1 [USD]
               Income groups are 1 (low) to 4 (high income).

climada/util/finance.py

@@ -24,8 +24,6 @@
 import json
 import logging
 import shutil
-import zipfile
-from pathlib import Path
 
 import numpy as np
 import pandas as pd
@@ -37,18 +35,7 @@
 
 LOGGER = logging.getLogger(__name__)
 
-WORLD_BANK_WEALTH_ACC = (
-    "https://databank.worldbank.org/data/download/Wealth-Accounts_CSV.zip"
-)
-"""Wealth historical data (1995, 2000, 2005, 2010, 2014) from World Bank (ZIP).
-    https://datacatalog.worldbank.org/dataset/wealth-accounting
-    Includes variable Produced Capital (NW.PCA.TO)"""
-
-FILE_WORLD_BANK_WEALTH_ACC = "Wealth-AccountsData.csv"
-
-WORLD_BANK_INC_GRP = (
-    "http://databank.worldbank.org/data/download/site-content/OGHIST.xls"
-)
+WORLD_BANK_INC_GRP = "https://ddh-openapi.worldbank.org/resources/DR0095334/download"
 """Income group historical data from World bank."""
 
 INCOME_GRP_WB_TABLE = {
@@ -73,6 +60,12 @@
 """File with wealth-to-GDP factors from the
 Credit Suisse's Global Wealth Report 2017 (household wealth)"""
 
+WB_URBAN_LAND_MARKUP = 1.24
+"""
+Markup factor of 24% used by the world bank to include built-value.
+See Section 9.2 [Worldbank methodology](https://datacatalog.worldbank.org/search/dataset/0042066/wealth-accounting)
+"""
+
 
 def _nat_earth_shp(resolution="10m", category="cultural", name="admin_0_countries"):
     shp_file = shapereader.natural_earth(
@@ -206,19 +199,20 @@ def download_world_bank_indicator(
     pages = np.inf
     page = 1
     while page <= pages:
-        response = requests.get(
+        url = (
             f"https://api.worldbank.org/v2/countries/{country_code}/indicators/"
-            f"{indicator}?format=json&page={page}",
-            timeout=30,
+            f"{indicator}?format=json&page={page}"
         )
+        response = requests.get(url, timeout=30)
         json_data = json.loads(response.text)
 
         # Check if we received an error message
         try:
             if json_data[0]["message"][0]["id"] == "120":
                 raise ValueError(
                     "Error requesting data from the World Bank API. Did you use the "
-                    "correct country code and indicator ID?"
+                    "correct country code and indicator ID?\n"
+                    f"{url}"
                 )
         # If no, we should be fine
         except KeyError:
@@ -415,9 +409,9 @@ def world_bank_wealth_account(
     cntry_iso, ref_year, variable_name="NW.PCA.TO", no_land=True
 ):
     """
-    Download and unzip wealth accounting historical data (1995, 2000, 2005, 2010, 2014)
-    from World Bank (https://datacatalog.worldbank.org/dataset/wealth-accounting).
-    Return requested variable for a country (cntry_iso) and a year (ref_year).
+    Download wealth accounting data from the World Bank API and return the requested
+    variable for a country (cntry_iso) and a year (ref_year).
+    https://datacatalog.worldbank.org/search/dataset/0042066
 
     Parameters
     ----------
@@ -426,10 +420,9 @@ def world_bank_wealth_account(
     ref_year : int
         reference year
 
-        * available in data: 1995, 2000, 2005, 2010, 2014
-        * other years between 1995 and 2014 are interpolated
-        * for years outside range, indicator is scaled
-          proportionally to GDP
+        * available in data: 1995-2020
+        * years within the data range are interpolated
+        * for years outside range, indicator is scaled proportionally to GDP
 
     variable_name : str
         select one variable, i.e.:
@@ -449,65 +442,60 @@ def world_bank_wealth_account(
             forests (timber and some nontimber forest products), and
             protected areas.
         'NW.TOW.TO': Total wealth of country.
-        Note: Values are measured at market exchange rates in constant 2014 US dollars,
-            using a country-specific GDP deflator.
+        Note: Values are measured at market exchange rates in
+              real chained 2019 US dollars.
+              See https://datacatalogfiles.worldbank.org/ddh-published-v2/0042066/9/DR0094582/CWON%202024%20Methodology_10122024.pdf
+
     no_land : boolean
         If True, return produced capital without built-up land value
-        (applies to 'NW.PCA.*' only). Default: True.
+        (applies to 'NW.PCA.*' only). The world bank applies a 24% markup
+        to include the built-up (or urban) land value, thus this divides
+        by a factor 1.24. (See section 9.2 of the worldbank methodology mentioned above).
+        Default: True.
     """
+
     try:
-        data_file = SYSTEM_DIR.joinpath(FILE_WORLD_BANK_WEALTH_ACC)
-        if not data_file.is_file():
-            data_file = SYSTEM_DIR.joinpath(
-                "Wealth-Accounts_CSV", FILE_WORLD_BANK_WEALTH_ACC
+        data_wealth = download_world_bank_indicator(
+            country_code=cntry_iso,
+            indicator=variable_name,
+        ).dropna()
+    except ValueError:
+        if "NW.PCA.TO" in variable_name:
+            LOGGER.warning(
+                "No data available for country. Using non-financial wealth instead"
             )
-        if not data_file.is_file():
-            if not SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").is_dir():
-                SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").mkdir()
-            file_down = download_file(WORLD_BANK_WEALTH_ACC)
-            zip_ref = zipfile.ZipFile(file_down, "r")
-            zip_ref.extractall(SYSTEM_DIR.joinpath("Wealth-Accounts_CSV"))
-            zip_ref.close()
-            Path(file_down).unlink()
-            LOGGER.debug("Download and unzip complete. Unzipping %s", str(data_file))
-
-        data_wealth = pd.read_csv(data_file, sep=",", index_col=None, header=0)
-    except Exception as err:
-        raise type(err)(
-            "Downloading World Bank Wealth Accounting Data failed: " + str(err)
-        ) from err
-
-    data_wealth = data_wealth[
-        data_wealth["Country Code"].str.contains(cntry_iso)
-        & data_wealth["Indicator Code"].str.contains(variable_name)
-    ].loc[:, "1995":"2014"]
-    years = list(map(int, list(data_wealth)))
-    if (
-        data_wealth.size == 0 and "NW.PCA.TO" in variable_name
-    ):  # if country is not found in data
-        LOGGER.warning(
-            "No data available for country. Using non-financial wealth instead"
-        )
-        gdp_year, gdp_val = gdp(cntry_iso, ref_year)
-        fac = wealth2gdp(cntry_iso)[1]
-        return gdp_year, np.around((fac * gdp_val), 1), 0
-    if ref_year in years:  # indicator for reference year is available directly
-        result = data_wealth.loc[:, str(ref_year)].values[0]
-    elif np.min(years) < ref_year < np.max(years):  # interpolate
-        result = np.interp(ref_year, years, data_wealth.values[0, :])
-    elif ref_year < np.min(years):  # scale proportionally to GDP
+            gdp_year, gdp_val = gdp(cntry_iso, ref_year)
+            fac = wealth2gdp(cntry_iso)[1]
+            return gdp_year, np.around((fac * gdp_val), 1), 0
+        else:
+            raise
+
+    years = data_wealth.index.values
+    if ref_year in years:
+        result = data_wealth.loc[ref_year]
+    elif np.min(years) < ref_year < np.max(years):
+        result = np.interp(ref_year, years, data_wealth.values)
+    elif ref_year < np.min(years):
         gdp_year, gdp0_val = gdp(cntry_iso, np.min(years))
         gdp_year, gdp_val = gdp(cntry_iso, ref_year)
-        result = data_wealth.values[0, 0] * gdp_val / gdp0_val
+        result = (
+            data_wealth.iloc[data_wealth.index.get_loc(np.min(years))]
+            * gdp_val
+            / gdp0_val
+        )
         ref_year = gdp_year
     else:
         gdp_year, gdp0_val = gdp(cntry_iso, np.max(years))
         gdp_year, gdp_val = gdp(cntry_iso, ref_year)
-        result = data_wealth.values[0, -1] * gdp_val / gdp0_val
+        result = (
+            data_wealth.iloc[data_wealth.index.get_loc(np.max(years))]
+            * gdp_val
+            / gdp0_val
+        )
         ref_year = gdp_year
+
     if "NW.PCA." in variable_name and no_land:
-        # remove value of built-up land from produced capital
-        result = result / 1.24
+        result = result / WB_URBAN_LAND_MARKUP
     return ref_year, np.around(result, 1), 1
 
 

climada/util/test/test_finance.py

@@ -25,6 +25,7 @@
 from cartopy.io import shapereader
 
 from climada.util.finance import (
+    WB_URBAN_LAND_MARKUP,
     _gdp_twn,
     download_world_bank_indicator,
     gdp,
@@ -202,33 +203,38 @@ def test_pca_DEU_2010_pass(self):
         """Test Processed Capital value Germany 2010."""
         ref_year = 2010
         cntry_iso = "DEU"
-        wb_year, wb_val, q = world_bank_wealth_account(cntry_iso, ref_year, no_land=0)
+        wb_year, wb_val, q = world_bank_wealth_account(
+            cntry_iso, ref_year, no_land=False
+        )
         wb_year_noland, wb_val_noland, q = world_bank_wealth_account(
-            cntry_iso, ref_year, no_land=1
+            cntry_iso, ref_year, no_land=True
         )
-        ref_val = [
-            17675048450284.9,
-            19767982562092.2,
-        ]  # second value as updated by worldbank on
-        # October 27 2021
-        ref_val_noland = [14254071330874.9, 15941921421042.1]  # dito
+        ref_val = np.array(
+            [
+                17675048450284.9,
+                19767982562092.2,  # October 27 2021
+                14676399165833.0,  # 15/05/2026 from WB website
+            ]
+        )  # Values updated by worldbank at different dates
+        ref_val_noland = ref_val / WB_URBAN_LAND_MARKUP
         self.assertEqual(wb_year, ref_year)
         self.assertEqual(q, 1)
         self.assertIn(wb_val, ref_val)
         self.assertEqual(wb_year_noland, ref_year)
-        self.assertIn(wb_val_noland, ref_val_noland)
+        assert any(np.isclose(wb_val_noland, ref_val) for ref_val in ref_val_noland)
 
     def test_pca_CHE_2008_pass(self):
         """Test Prcoessed Capital per capita Switzerland 2008 (interp.)."""
         ref_year = 2008
         cntry_iso = "CHE"
         var_name = "NW.PCA.PC"
         wb_year, wb_val, _ = world_bank_wealth_account(
-            cntry_iso, ref_year, variable_name=var_name, no_land=0
+            cntry_iso, ref_year, variable_name=var_name, no_land=False
         )
         ref_val = [
             328398.7,  # values sporadically updated by worldbank
             369081.0,
+            357588.0,  # Value checked on 15/05/2026 from WB website
         ]  # <- October 27 2021
         self.assertEqual(wb_year, ref_year)
         self.assertIn(wb_val, ref_val)
@@ -246,6 +252,7 @@ def test_tow_IND_1985_pass(self):
             5861193808779.6,  # <- October 27 2021
             5861186556152.8,  # <- June 29 2023
             5861186367245.2,  # <- December 20 2023
+            7925612508833.5,  # <- 15/05/2026 (changed from 2014 constant $ to 2019 real chained $)
         ]
         self.assertEqual(wb_year, ref_year)
         self.assertIn(wb_val, ref_val)
@@ -254,7 +261,9 @@ def test_pca_CUB_2015_pass(self):
         """Test Processed Capital value Cuba 2015 (missing value)."""
         ref_year = 2015
         cntry_iso = "CUB"
-        wb_year, wb_val, q = world_bank_wealth_account(cntry_iso, ref_year, no_land=1)
+        wb_year, wb_val, q = world_bank_wealth_account(
+            cntry_iso, ref_year, no_land=True
+        )
         ref_val = [
             108675762920.0,  # values sporadically updated by worldbank
             108675513472.0,  # <- Dezember 20 2023