From a8a9136427bcd6d823576c953d044ceaad9efcd7 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Thu, 23 Jul 2026 18:45:54 +0300 Subject: [PATCH 1/5] Default decile impacts to household net income --- changelog.d/485.fixed.md | 1 + docs/impact-analysis.md | 2 +- docs/outputs.md | 2 +- src/policyengine/outputs/decile_impact.py | 13 ++-- .../tax_benefit_models/uk/analysis.py | 3 + tests/test_intra_decile_impact.py | 78 ++++++++++++++++++- tests/test_uk_analysis.py | 1 + 7 files changed, 88 insertions(+), 12 deletions(-) create mode 100644 changelog.d/485.fixed.md diff --git a/changelog.d/485.fixed.md b/changelog.d/485.fixed.md new file mode 100644 index 00000000..dc5ccaab --- /dev/null +++ b/changelog.d/485.fixed.md @@ -0,0 +1 @@ +Default decile-impact calculations to household net income while preserving explicit selection of equivalised HBAI net income. diff --git a/docs/impact-analysis.md b/docs/impact-analysis.md index 275dc14c..64797b28 100644 --- a/docs/impact-analysis.md +++ b/docs/impact-analysis.md @@ -31,7 +31,7 @@ A `PolicyReformAnalysis` with: | Attribute | Type | Content | |---|---|---| -| `decile_impacts` | `OutputCollection[DecileImpact]` | Mean baseline / reform / change and winner-loser counts per decile | +| `decile_impacts` | `OutputCollection[DecileImpact]` | Household net income baseline / reform / change and winner-loser counts per income decile | | `wealth_decile_impacts` | `OutputCollection[DecileImpact]` | UK only: household net income impacts grouped by `household_wealth_decile` | | `intra_wealth_decile_impacts` | `OutputCollection[IntraDecileImpact]` | UK only: within-wealth-decile distribution of household net income changes | | `program_statistics` | `OutputCollection[ProgramStatistics]` | Totals, counts, winners/losers per program | diff --git a/docs/outputs.md b/docs/outputs.md index 2bab1cd1..c2ce0f94 100644 --- a/docs/outputs.md +++ b/docs/outputs.md @@ -96,7 +96,7 @@ Or on a relative change — `relative_change_geq=0.05` selects households with a One decile's baseline mean, reform mean, and mean change. For all ten at once, use `calculate_decile_impacts`. -By default, `calculate_decile_impacts` ranks units into deciles using `income_variable`. To measure changes in one variable while grouping by an existing decile variable, pass `decile_variable`. For example, UK wealth-decile impacts measure changes in household net income grouped by `household_wealth_decile`. +By default, `calculate_decile_impacts` measures `household_net_income` and ranks units into deciles using that variable. Pass another `income_variable`, such as `equiv_hbai_household_net_income`, to select a different income concept explicitly. To measure changes in one variable while grouping by an existing decile variable, pass `decile_variable`. For example, UK wealth-decile impacts measure changes in household net income grouped by `household_wealth_decile`. ```python from policyengine.outputs import calculate_decile_impacts diff --git a/src/policyengine/outputs/decile_impact.py b/src/policyengine/outputs/decile_impact.py index 3f19931a..d2dcd9a8 100644 --- a/src/policyengine/outputs/decile_impact.py +++ b/src/policyengine/outputs/decile_impact.py @@ -17,7 +17,7 @@ class DecileImpact(Output): baseline_simulation: Simulation reform_simulation: Simulation - income_variable: str = "equiv_hbai_household_net_income" + income_variable: str = "household_net_income" decile_variable: Optional[str] = None # If set, use pre-computed grouping variable entity: Optional[str] = None decile: int @@ -104,7 +104,7 @@ def calculate_decile_impacts( baseline_policy: Optional[Policy] = None, reform_policy: Optional[Policy] = None, dynamic: Optional[Dynamic] = None, - income_variable: str = "equiv_hbai_household_net_income", + income_variable: str = "household_net_income", decile_variable: Optional[str] = None, entity: Optional[str] = None, quantiles: int = 10, @@ -113,10 +113,11 @@ def calculate_decile_impacts( ) -> OutputCollection[DecileImpact]: """Calculate decile-by-decile impact of a reform. - By default, deciles are computed from ``income_variable``. Pass - ``decile_variable`` to group by a pre-computed decile variable while - still measuring changes in ``income_variable``; for example, UK wealth - deciles use ``income_variable="household_net_income"`` with + By default, changes are measured in ``household_net_income`` and deciles + are computed from that variable. Pass ``decile_variable`` to group by a + pre-computed decile variable while still measuring changes in + ``income_variable``; for example, UK wealth deciles use + ``income_variable="household_net_income"`` with ``decile_variable="household_wealth_decile"``. Returns: diff --git a/src/policyengine/tax_benefit_models/uk/analysis.py b/src/policyengine/tax_benefit_models/uk/analysis.py index 266736fc..7460565b 100644 --- a/src/policyengine/tax_benefit_models/uk/analysis.py +++ b/src/policyengine/tax_benefit_models/uk/analysis.py @@ -115,9 +115,12 @@ def economic_impact_analysis( "Reform simulation must have more than 100 households" ) + # Keep both UK decile configurations explicit so changes to shared output + # defaults do not silently alter the country analysis bundle. decile_impacts = calculate_decile_impacts( baseline_simulation=baseline_simulation, reform_simulation=reform_simulation, + income_variable="household_net_income", ) wealth_decile_impacts = calculate_decile_impacts( baseline_simulation=baseline_simulation, diff --git a/tests/test_intra_decile_impact.py b/tests/test_intra_decile_impact.py index 8f8960c1..27232ec9 100644 --- a/tests/test_intra_decile_impact.py +++ b/tests/test_intra_decile_impact.py @@ -367,8 +367,19 @@ def test_decile_impact_qcut_default(): assert abs(di.absolute_change - 1000.0) < 1e-6 +def test_decile_impact_defaults_to_household_net_income(): + """DecileImpact measures household net income unless configured otherwise.""" + impact = DecileImpact.model_construct( + baseline_simulation=MagicMock(), + reform_simulation=MagicMock(), + decile=1, + ) + + assert impact.income_variable == "household_net_income" + + def test_calculate_decile_impacts_uses_supplied_simulations(monkeypatch): - """calculate_decile_impacts can reuse simulations that already have outputs.""" + """The helper reuses simulations and defaults to household net income.""" version = _make_version("household_net_income", "household") baseline = Simulation.model_construct( tax_benefit_model_version=version, @@ -425,24 +436,80 @@ def fake_ensure(self): results = calculate_decile_impacts( baseline_simulation=baseline, reform_simulation=reform, - income_variable="household_net_income", entity="household", quantiles=2, ) assert len(ensure_calls) == 2 assert len(results.outputs) == 2 + assert all( + result.income_variable == "household_net_income" for result in results.outputs + ) assert all( abs(result.absolute_change - 1000.0) < 1e-6 for result in results.outputs ) +def test_calculate_decile_impacts_accepts_explicit_equiv_hbai_income(monkeypatch): + """Callers can explicitly select equivalised HBAI net income.""" + variable = "equiv_hbai_household_net_income" + version = _make_version(variable, "household") + baseline = Simulation.model_construct( + tax_benefit_model_version=version, + output_dataset=MagicMock( + data=MagicMock( + household=MicroDataFrame( + pd.DataFrame( + { + variable: [10000.0, 20000.0, 30000.0, 40000.0], + "household_weight": [1.0, 1.0, 1.0, 1.0], + } + ), + weights="household_weight", + ) + ) + ), + ) + reform = Simulation.model_construct( + tax_benefit_model_version=version, + output_dataset=MagicMock( + data=MagicMock( + household=MicroDataFrame( + pd.DataFrame( + { + variable: [10500.0, 20500.0, 30500.0, 40500.0], + "household_weight": [1.0, 1.0, 1.0, 1.0], + } + ), + weights="household_weight", + ) + ) + ), + ) + + monkeypatch.setattr( + "policyengine.outputs.decile_impact.Simulation.ensure", + lambda self: None, + ) + + results = calculate_decile_impacts( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable=variable, + entity="household", + quantiles=2, + ) + + assert all(result.income_variable == variable for result in results.outputs) + assert all(result.absolute_change == 500.0 for result in results.outputs) + + def test_calculate_decile_impacts_ensures_constructed_simulations( uk_test_dataset, monkeypatch ): """calculate_decile_impacts populates outputs when constructing simulations internally.""" household_df = pd.DataFrame(uk_test_dataset.data.household) - household_df["equiv_hbai_household_net_income"] = [ + household_df["household_net_income"] = [ 10000.0, 20000.0, 30000.0, @@ -466,7 +533,7 @@ def fake_ensure(self): results = calculate_decile_impacts( dataset=uk_test_dataset, tax_benefit_model_version=_make_version( - "equiv_hbai_household_net_income", + "household_net_income", "household", ), quantiles=3, @@ -474,4 +541,7 @@ def fake_ensure(self): assert len(ensure_calls) == 2 assert len(results.outputs) == 3 + assert all( + result.income_variable == "household_net_income" for result in results.outputs + ) assert all(abs(result.absolute_change) < 1e-9 for result in results.outputs) diff --git a/tests/test_uk_analysis.py b/tests/test_uk_analysis.py index 85c22c80..7aad0ac2 100644 --- a/tests/test_uk_analysis.py +++ b/tests/test_uk_analysis.py @@ -121,6 +121,7 @@ def fake_inequality(simulation): standard_call = decile_calls[0] assert standard_call["baseline_simulation"] is baseline assert standard_call["reform_simulation"] is reform + assert standard_call["income_variable"] == "household_net_income" wealth_call = decile_calls[1] assert wealth_call["baseline_simulation"] is baseline From a289cc49731ff375f54d5c0a6f5fc817c54d833d Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Fri, 24 Jul 2026 02:06:50 +0300 Subject: [PATCH 2/5] Use person-weighted decile groups --- changelog.d/485.fixed.md | 2 +- docs/impact-analysis.md | 2 +- docs/outputs.md | 4 +- pyproject.toml | 2 +- src/policyengine/outputs/decile_grouping.py | 61 ++++++++++++ src/policyengine/outputs/decile_impact.py | 30 +++--- .../outputs/intra_decile_impact.py | 32 +++--- tests/test_decile_grouping.py | 98 +++++++++++++++++++ tests/test_intra_decile_impact.py | 11 ++- uv.lock | 10 +- 10 files changed, 206 insertions(+), 46 deletions(-) create mode 100644 src/policyengine/outputs/decile_grouping.py create mode 100644 tests/test_decile_grouping.py diff --git a/changelog.d/485.fixed.md b/changelog.d/485.fixed.md index dc5ccaab..7a7304b2 100644 --- a/changelog.d/485.fixed.md +++ b/changelog.d/485.fixed.md @@ -1 +1 @@ -Default decile-impact calculations to household net income while preserving explicit selection of equivalised HBAI net income. +Default decile-impact calculations to household net income and person-weight computed household groups, while preserving explicit selection of equivalised HBAI net income. diff --git a/docs/impact-analysis.md b/docs/impact-analysis.md index 64797b28..301a322f 100644 --- a/docs/impact-analysis.md +++ b/docs/impact-analysis.md @@ -31,7 +31,7 @@ A `PolicyReformAnalysis` with: | Attribute | Type | Content | |---|---|---| -| `decile_impacts` | `OutputCollection[DecileImpact]` | Household net income baseline / reform / change and winner-loser counts per income decile | +| `decile_impacts` | `OutputCollection[DecileImpact]` | Household net income baseline / reform / change and winner-loser counts per person-weighted income decile | | `wealth_decile_impacts` | `OutputCollection[DecileImpact]` | UK only: household net income impacts grouped by `household_wealth_decile` | | `intra_wealth_decile_impacts` | `OutputCollection[IntraDecileImpact]` | UK only: within-wealth-decile distribution of household net income changes | | `program_statistics` | `OutputCollection[ProgramStatistics]` | Totals, counts, winners/losers per program | diff --git a/docs/outputs.md b/docs/outputs.md index c2ce0f94..1387bcaf 100644 --- a/docs/outputs.md +++ b/docs/outputs.md @@ -96,7 +96,7 @@ Or on a relative change — `relative_change_geq=0.05` selects households with a One decile's baseline mean, reform mean, and mean change. For all ten at once, use `calculate_decile_impacts`. -By default, `calculate_decile_impacts` measures `household_net_income` and ranks units into deciles using that variable. Pass another `income_variable`, such as `equiv_hbai_household_net_income`, to select a different income concept explicitly. To measure changes in one variable while grouping by an existing decile variable, pass `decile_variable`. For example, UK wealth-decile impacts measure changes in household net income grouped by `household_wealth_decile`. +By default, `calculate_decile_impacts` measures `household_net_income` and ranks units into deciles using that variable. Household groups are person-weighted: their survey weights are multiplied by `household_count_people` before ranking. Pass another `income_variable`, such as `equiv_hbai_household_net_income`, to select a different income concept explicitly. To measure changes in one variable while grouping by an existing decile variable, pass `decile_variable`. For example, UK wealth-decile impacts measure changes in household net income grouped by `household_wealth_decile`. ```python from policyengine.outputs import calculate_decile_impacts @@ -123,7 +123,7 @@ impacts.dataframe # includes the decile_variable column ## IntraDecileImpact -Distribution of household-level impact within each decile (five bucket categories summing to 1.0). Use `compute_intra_decile_impacts` for the full set. Like `calculate_decile_impacts`, this helper accepts `decile_variable` when the grouping variable is already present in the simulation output. +Distribution of household-level impact within each decile (five bucket categories summing to 1.0). Use `compute_intra_decile_impacts` for the full set. Its computed household groups use the same person-weighted ranking as `calculate_decile_impacts`, and its category proportions are also person-weighted. This helper accepts `decile_variable` when the grouping variable is already present in the simulation output. ```python from policyengine.outputs import compute_intra_decile_impacts diff --git a/pyproject.toml b/pyproject.toml index 8ae2dcd9..db92e4b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "pydantic>=2.0.0", "pandas>=2.0.0", "h5py>=3.0.0", - "microdf_python>=1.2.1", + "microdf_python>=1.3.0", "jsonschema>=4.0.0", "requests>=2.31.0", "psutil>=5.9.0", diff --git a/src/policyengine/outputs/decile_grouping.py b/src/policyengine/outputs/decile_grouping.py new file mode 100644 index 00000000..0df566f4 --- /dev/null +++ b/src/policyengine/outputs/decile_grouping.py @@ -0,0 +1,61 @@ +"""Shared weighted grouping for decile-based outputs.""" + +from typing import Any, Optional + +import numpy as np +import pandas as pd +from microdf import MicroSeries + + +def calculate_decile_groups( + baseline_data: Any, + ranking_values: Any, + *, + decile_variable: Optional[str], + entity: str, + quantiles: int, +) -> Any: + """Return precomputed groups or weighted ranks of ``ranking_values``. + + Household income groups follow the convention used by both country + packages: survey weights are multiplied by household size so that each + decile represents an approximately equal number of people. Other entities + use their entity survey weights without an additional multiplier. + """ + + if decile_variable: + return baseline_data[decile_variable] + if quantiles < 1: + raise ValueError("quantiles must be at least 1") + + weight_variable = f"{entity}_weight" + if weight_variable not in baseline_data.columns: + raise ValueError( + f"Weighted quantile grouping requires '{weight_variable}' in " + f"baseline output data for entity '{entity}'." + ) + weights = np.asarray(baseline_data[weight_variable], dtype=float) + + if entity == "household": + multiplier_variable = "household_count_people" + if multiplier_variable not in baseline_data.columns: + raise ValueError( + "Person-weighted household quantile grouping requires " + "'household_count_people' in baseline output data." + ) + weights = weights * np.asarray( + baseline_data[multiplier_variable], + dtype=float, + ) + + weighted_values = MicroSeries( + np.asarray(ranking_values), + index=baseline_data.index, + weights=weights, + ) + percentile_ranks = np.asarray(weighted_values.rank(pct=True)) + groups = np.minimum( + np.ceil(percentile_ranks * quantiles), + quantiles, + ).astype(int) + return pd.Series(groups, index=baseline_data.index) diff --git a/src/policyengine/outputs/decile_impact.py b/src/policyengine/outputs/decile_impact.py index d2dcd9a8..d9f7a608 100644 --- a/src/policyengine/outputs/decile_impact.py +++ b/src/policyengine/outputs/decile_impact.py @@ -8,6 +8,7 @@ from policyengine.core.dynamic import Dynamic from policyengine.core.policy import Policy from policyengine.core.tax_benefit_model_version import TaxBenefitModelVersion +from policyengine.outputs.decile_grouping import calculate_decile_groups class DecileImpact(Output): @@ -67,19 +68,13 @@ def run(self): baseline_income = baseline_data[self.income_variable] reform_income = reform_data[self.income_variable] - # Calculate deciles: use pre-computed variable or qcut - if self.decile_variable: - decile_series = baseline_data[self.decile_variable] - else: - decile_series = ( - pd.qcut( - baseline_income, - self.quantiles, - labels=False, - duplicates="drop", - ) - + 1 - ) + decile_series = calculate_decile_groups( + baseline_data, + baseline_income, + decile_variable=self.decile_variable, + entity=target_entity, + quantiles=self.quantiles, + ) # Calculate changes absolute_change = reform_income - baseline_income @@ -113,10 +108,11 @@ def calculate_decile_impacts( ) -> OutputCollection[DecileImpact]: """Calculate decile-by-decile impact of a reform. - By default, changes are measured in ``household_net_income`` and deciles - are computed from that variable. Pass ``decile_variable`` to group by a - pre-computed decile variable while still measuring changes in - ``income_variable``; for example, UK wealth deciles use + By default, changes are measured in ``household_net_income`` and household + deciles are computed from that variable using survey weights multiplied by + household size. Pass ``decile_variable`` to group by a pre-computed decile + variable while still measuring changes in ``income_variable``; for example, + UK wealth deciles use ``income_variable="household_net_income"`` with ``decile_variable="household_wealth_decile"``. diff --git a/src/policyengine/outputs/intra_decile_impact.py b/src/policyengine/outputs/intra_decile_impact.py index b91a04e2..bfdb6354 100644 --- a/src/policyengine/outputs/intra_decile_impact.py +++ b/src/policyengine/outputs/intra_decile_impact.py @@ -22,6 +22,7 @@ from pydantic import ConfigDict from policyengine.core import Output, OutputCollection, Simulation +from policyengine.outputs.decile_grouping import calculate_decile_groups # The 5-category thresholds BOUNDS = [-np.inf, -0.05, -1e-3, 1e-3, 0.05, np.inf] @@ -62,27 +63,24 @@ def run(self): ) reform_data = getattr(self.reform_simulation.output_dataset.data, self.entity) - baseline_income = baseline_data[self.income_variable].values - reform_income = reform_data[self.income_variable].values - - # Determine decile grouping - if self.decile_variable: - decile_series = baseline_data[self.decile_variable].values - else: - decile_series = ( - pd.qcut( - baseline_income, - self.quantiles, - labels=False, - duplicates="drop", - ) - + 1 + baseline_income_series = baseline_data[self.income_variable] + reform_income_series = reform_data[self.income_variable] + decile_series = np.asarray( + calculate_decile_groups( + baseline_data, + baseline_income_series, + decile_variable=self.decile_variable, + entity=self.entity, + quantiles=self.quantiles, ) + ) + baseline_income = np.asarray(baseline_income_series) + reform_income = np.asarray(reform_income_series) # People-weighted counts - weights = baseline_data[f"{self.entity}_weight"].values + weights = np.asarray(baseline_data[f"{self.entity}_weight"]) if self.entity == "household": - people_count = baseline_data["household_count_people"].values + people_count = np.asarray(baseline_data["household_count_people"]) people = people_count * weights else: people = weights diff --git a/tests/test_decile_grouping.py b/tests/test_decile_grouping.py new file mode 100644 index 00000000..ff79aa2b --- /dev/null +++ b/tests/test_decile_grouping.py @@ -0,0 +1,98 @@ +"""Tests for shared weighted decile grouping.""" + +import pandas as pd +import pytest +from microdf import MicroDataFrame + +from policyengine.outputs.decile_grouping import calculate_decile_groups + + +def _household_frame( + incomes, + household_weights, + people_counts=None, + *, + index=None, +) -> MicroDataFrame: + data = { + "household_net_income": incomes, + "household_weight": household_weights, + } + if people_counts is not None: + data["household_count_people"] = people_counts + return MicroDataFrame( + pd.DataFrame(data, index=index), + weights="household_weight", + ) + + +def test_household_groups_multiply_survey_weights_by_household_size(): + household = _household_frame( + [10, 20, 30, 40], + [2, 1, 1, 1], + [2, 1, 1, 1], + index=[100, 200, 300, 400], + ) + + groups = calculate_decile_groups( + household, + household["household_net_income"], + decile_variable=None, + entity="household", + quantiles=10, + ) + + assert groups.index.tolist() == [100, 200, 300, 400] + assert groups.tolist() == [6, 8, 9, 10] + + +def test_household_groups_keep_tied_incomes_together(): + household = _household_frame( + [10, 10, 20], + [1, 1, 1], + [1, 2, 1], + ) + + groups = calculate_decile_groups( + household, + household["household_net_income"], + decile_variable=None, + entity="household", + quantiles=10, + ) + + assert groups.tolist() == [8, 8, 10] + + +def test_precomputed_groups_bypass_weighted_ranking_requirements(): + household = _household_frame( + [10, 20], + [1, 1], + ) + household["household_wealth_decile"] = [2, 1] + + groups = calculate_decile_groups( + household, + household["household_net_income"], + decile_variable="household_wealth_decile", + entity="household", + quantiles=10, + ) + + assert groups.tolist() == [2, 1] + + +def test_computed_household_groups_require_people_counts(): + household = _household_frame( + [10, 20], + [1, 1], + ) + + with pytest.raises(ValueError, match="household_count_people"): + calculate_decile_groups( + household, + household["household_net_income"], + decile_variable=None, + entity="household", + quantiles=10, + ) diff --git a/tests/test_intra_decile_impact.py b/tests/test_intra_decile_impact.py index 27232ec9..4b4cf08c 100644 --- a/tests/test_intra_decile_impact.py +++ b/tests/test_intra_decile_impact.py @@ -329,8 +329,8 @@ def test_calculate_decile_impacts_with_decile_variable(monkeypatch): ] -def test_decile_impact_qcut_default(): - """Without decile_variable, DecileImpact uses qcut (default behavior).""" +def test_decile_impact_weighted_grouping_default(): + """Without decile_variable, DecileImpact computes weighted groups.""" n = 100 incomes = np.linspace(10000, 100000, n) reform_incomes = incomes + 1000 @@ -340,6 +340,7 @@ def test_decile_impact_qcut_default(): { "household_net_income": incomes, "household_weight": np.ones(n), + "household_count_people": np.ones(n), }, variables=variables, ) @@ -347,6 +348,7 @@ def test_decile_impact_qcut_default(): { "household_net_income": reform_incomes, "household_weight": np.ones(n), + "household_count_people": np.ones(n), }, variables=variables, ) @@ -395,6 +397,7 @@ def test_calculate_decile_impacts_uses_supplied_simulations(monkeypatch): 100000.0, ], "household_weight": [1.0, 1.0, 1.0, 1.0], + "household_count_people": [1.0, 1.0, 1.0, 1.0], } ), weights="household_weight", @@ -416,6 +419,7 @@ def test_calculate_decile_impacts_uses_supplied_simulations(monkeypatch): 101000.0, ], "household_weight": [1.0, 1.0, 1.0, 1.0], + "household_count_people": [1.0, 1.0, 1.0, 1.0], } ), weights="household_weight", @@ -463,6 +467,7 @@ def test_calculate_decile_impacts_accepts_explicit_equiv_hbai_income(monkeypatch { variable: [10000.0, 20000.0, 30000.0, 40000.0], "household_weight": [1.0, 1.0, 1.0, 1.0], + "household_count_people": [1.0, 1.0, 1.0, 1.0], } ), weights="household_weight", @@ -479,6 +484,7 @@ def test_calculate_decile_impacts_accepts_explicit_equiv_hbai_income(monkeypatch { variable: [10500.0, 20500.0, 30500.0, 40500.0], "household_weight": [1.0, 1.0, 1.0, 1.0], + "household_count_people": [1.0, 1.0, 1.0, 1.0], } ), weights="household_weight", @@ -514,6 +520,7 @@ def test_calculate_decile_impacts_ensures_constructed_simulations( 20000.0, 30000.0, ] + household_df["household_count_people"] = [2.0, 2.0, 2.0] uk_test_dataset.data.household = MicroDataFrame( household_df, weights="household_weight", diff --git a/uv.lock b/uv.lock index 66bc83bb..b12d83f5 100644 --- a/uv.lock +++ b/uv.lock @@ -2064,16 +2064,16 @@ wheels = [ [[package]] name = "microdf-python" -version = "1.2.1" +version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "numpy", version = "2.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "pandas" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e7/96/6f9f37f79f2c6440d91036a7bf8111dd4b983c577a7e96d45bf3ca4171f3/microdf_python-1.2.1.tar.gz", hash = "sha256:d4f58e4e0c21decd0c6d425b115db8acc72751c558f48d2a1c3a6619f168a94a", size = 19641, upload-time = "2026-01-25T13:40:57.147Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/1b/a4b6a53c8e6668c190e1928ebf9f8c67d6a4995ff4a4b0e3403ded30d4ab/microdf_python-1.3.0.tar.gz", hash = "sha256:f352fe32c25a4fd6d6188ef6aaca0214b53b29a8a7913aca6af0362dd9c922bd", size = 28141, upload-time = "2026-04-17T23:20:57.018Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cd/2e/375ab71f8d91b691597247b186a4d7b156d2ed975dfb00450e560beae747/microdf_python-1.2.1-py3-none-any.whl", hash = "sha256:3c3d318a82cba7db0ef5a72e8a73a6072fe0bc7a9cb59b1eac01a26ee8c82e7c", size = 20879, upload-time = "2026-01-25T13:40:55.877Z" }, + { url = "https://files.pythonhosted.org/packages/b9/aa/ce692e0488b5e6c0943f52886cf6a15f5d9a6dee7744b190a3a1d67f5d6e/microdf_python-1.3.0-py3-none-any.whl", hash = "sha256:e17757a198648108eae24f39384e27b5ae8061acf1b43fa56520010dc00a7b55", size = 28908, upload-time = "2026-04-17T23:20:56.051Z" }, ] [[package]] @@ -2820,7 +2820,7 @@ wheels = [ [[package]] name = "policyengine" -version = "4.22.0" +version = "4.22.2" source = { editable = "." } dependencies = [ { name = "diskcache" }, @@ -2890,7 +2890,7 @@ requires-dist = [ { name = "itables", marker = "extra == 'dev'" }, { name = "jsonschema", specifier = ">=4.0.0" }, { name = "jupyter-book", marker = "extra == 'dev'" }, - { name = "microdf-python", specifier = ">=1.2.1" }, + { name = "microdf-python", specifier = ">=1.3.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.11.0" }, { name = "networkx", marker = "extra == 'graph'", specifier = ">=3.0" }, { name = "packaging", specifier = ">=23.0" }, From 99dc907db7c1279d6625a0125b447b94ca742989 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Fri, 24 Jul 2026 17:02:06 +0300 Subject: [PATCH 3/5] Address weighted decile review feedback --- changelog.d/485.breaking.md | 1 + changelog.d/485.fixed.md | 1 - docs/impact-analysis.md | 2 +- docs/outputs.md | 2 +- pyproject.toml | 2 +- src/policyengine/outputs/decile_grouping.py | 15 ++++-- src/policyengine/outputs/decile_impact.py | 9 ++-- tests/test_decile_grouping.py | 54 +++++++++++++++++++++ uv.lock | 2 +- 9 files changed, 77 insertions(+), 11 deletions(-) create mode 100644 changelog.d/485.breaking.md delete mode 100644 changelog.d/485.fixed.md diff --git a/changelog.d/485.breaking.md b/changelog.d/485.breaking.md new file mode 100644 index 00000000..7428fcd8 --- /dev/null +++ b/changelog.d/485.breaking.md @@ -0,0 +1 @@ +Changed default decile-impact calculations to household net income and person-weighted household groups, excluded negative computed-income values from reported deciles, and preserved explicit selection of equivalised HBAI net income. diff --git a/changelog.d/485.fixed.md b/changelog.d/485.fixed.md deleted file mode 100644 index 7a7304b2..00000000 --- a/changelog.d/485.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Default decile-impact calculations to household net income and person-weight computed household groups, while preserving explicit selection of equivalised HBAI net income. diff --git a/docs/impact-analysis.md b/docs/impact-analysis.md index 301a322f..9063db68 100644 --- a/docs/impact-analysis.md +++ b/docs/impact-analysis.md @@ -31,7 +31,7 @@ A `PolicyReformAnalysis` with: | Attribute | Type | Content | |---|---|---| -| `decile_impacts` | `OutputCollection[DecileImpact]` | Household net income baseline / reform / change and winner-loser counts per person-weighted income decile | +| `decile_impacts` | `OutputCollection[DecileImpact]` | Household net income baseline / reform / change and winner-loser counts per person-weighted income decile; negative-income rows are excluded from reported deciles | | `wealth_decile_impacts` | `OutputCollection[DecileImpact]` | UK only: household net income impacts grouped by `household_wealth_decile` | | `intra_wealth_decile_impacts` | `OutputCollection[IntraDecileImpact]` | UK only: within-wealth-decile distribution of household net income changes | | `program_statistics` | `OutputCollection[ProgramStatistics]` | Totals, counts, winners/losers per program | diff --git a/docs/outputs.md b/docs/outputs.md index 1387bcaf..15ca1f27 100644 --- a/docs/outputs.md +++ b/docs/outputs.md @@ -96,7 +96,7 @@ Or on a relative change — `relative_change_geq=0.05` selects households with a One decile's baseline mean, reform mean, and mean change. For all ten at once, use `calculate_decile_impacts`. -By default, `calculate_decile_impacts` measures `household_net_income` and ranks units into deciles using that variable. Household groups are person-weighted: their survey weights are multiplied by `household_count_people` before ranking. Pass another `income_variable`, such as `equiv_hbai_household_net_income`, to select a different income concept explicitly. To measure changes in one variable while grouping by an existing decile variable, pass `decile_variable`. For example, UK wealth-decile impacts measure changes in household net income grouped by `household_wealth_decile`. +By default, `calculate_decile_impacts` measures `household_net_income` and ranks units into deciles using that variable. Household groups are person-weighted: their survey weights are multiplied by `household_count_people` before ranking. Rows with negative values of the computed income concept receive the conventional `-1` group and are excluded from the reported decile results. Pass another `income_variable`, such as `equiv_hbai_household_net_income`, to select a different income concept explicitly. To measure changes in one variable while grouping by an existing decile variable, pass `decile_variable`. Precomputed group values outside `1..quantiles` are excluded, which preserves country-package sentinel values such as `-1` for negative income or wealth. For example, UK wealth-decile impacts measure changes in household net income grouped by `household_wealth_decile`. ```python from policyengine.outputs import calculate_decile_impacts diff --git a/pyproject.toml b/pyproject.toml index db92e4b5..fa3a1074 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "pydantic>=2.0.0", "pandas>=2.0.0", "h5py>=3.0.0", - "microdf_python>=1.3.0", + "microdf_python==1.3.0", "jsonschema>=4.0.0", "requests>=2.31.0", "psutil>=5.9.0", diff --git a/src/policyengine/outputs/decile_grouping.py b/src/policyengine/outputs/decile_grouping.py index 0df566f4..9e462637 100644 --- a/src/policyengine/outputs/decile_grouping.py +++ b/src/policyengine/outputs/decile_grouping.py @@ -20,7 +20,13 @@ def calculate_decile_groups( Household income groups follow the convention used by both country packages: survey weights are multiplied by household size so that each decile represents an approximately equal number of people. Other entities - use their entity survey weights without an additional multiplier. + use their entity survey weights without an additional multiplier. Negative + ranking values are assigned ``-1`` and therefore excluded from reported + groups, matching the country-package income-decile convention. + + Precomputed groups are returned unchanged. Callers that use values outside + ``1..quantiles`` (including the conventional ``-1`` sentinel) can therefore + intentionally exclude rows from reported groups. """ if decile_variable: @@ -48,14 +54,17 @@ def calculate_decile_groups( dtype=float, ) + ranking_array = np.asarray(ranking_values) weighted_values = MicroSeries( - np.asarray(ranking_values), + ranking_array, index=baseline_data.index, weights=weights, ) percentile_ranks = np.asarray(weighted_values.rank(pct=True)) - groups = np.minimum( + groups = np.clip( np.ceil(percentile_ranks * quantiles), + 1, quantiles, ).astype(int) + groups[ranking_array < 0] = -1 return pd.Series(groups, index=baseline_data.index) diff --git a/src/policyengine/outputs/decile_impact.py b/src/policyengine/outputs/decile_impact.py index d9f7a608..02ba0ab7 100644 --- a/src/policyengine/outputs/decile_impact.py +++ b/src/policyengine/outputs/decile_impact.py @@ -110,9 +110,12 @@ def calculate_decile_impacts( By default, changes are measured in ``household_net_income`` and household deciles are computed from that variable using survey weights multiplied by - household size. Pass ``decile_variable`` to group by a pre-computed decile - variable while still measuring changes in ``income_variable``; for example, - UK wealth deciles use + household size. Households with negative values of the computed income + concept are excluded from the reported deciles, matching country-package + income-decile outputs. Pass ``decile_variable`` to group by a pre-computed + decile variable while still measuring changes in ``income_variable``; + values outside ``1..quantiles`` are excluded from the reported groups. For + example, UK wealth deciles use ``income_variable="household_net_income"`` with ``decile_variable="household_wealth_decile"``. diff --git a/tests/test_decile_grouping.py b/tests/test_decile_grouping.py index ff79aa2b..f4001eb4 100644 --- a/tests/test_decile_grouping.py +++ b/tests/test_decile_grouping.py @@ -64,6 +64,42 @@ def test_household_groups_keep_tied_incomes_together(): assert groups.tolist() == [8, 8, 10] +def test_computed_groups_clamp_zero_weight_rows_to_first_group(): + household = _household_frame( + [10, 20, 30], + [0, 1, 1], + [1, 1, 1], + ) + + groups = calculate_decile_groups( + household, + household["household_net_income"], + decile_variable=None, + entity="household", + quantiles=10, + ) + + assert groups.tolist() == [1, 5, 10] + + +def test_computed_groups_exclude_negative_ranking_values(): + household = _household_frame( + [-10, 20, 30], + [1, 1, 1], + [1, 1, 1], + ) + + groups = calculate_decile_groups( + household, + household["household_net_income"], + decile_variable=None, + entity="household", + quantiles=10, + ) + + assert groups.tolist() == [-1, 7, 10] + + def test_precomputed_groups_bypass_weighted_ranking_requirements(): household = _household_frame( [10, 20], @@ -82,6 +118,24 @@ def test_precomputed_groups_bypass_weighted_ranking_requirements(): assert groups.tolist() == [2, 1] +def test_precomputed_groups_preserve_exclusion_sentinel(): + household = _household_frame( + [10, 20], + [1, 1], + ) + household["household_income_decile"] = [-1, 2] + + groups = calculate_decile_groups( + household, + household["household_net_income"], + decile_variable="household_income_decile", + entity="household", + quantiles=10, + ) + + assert groups.tolist() == [-1, 2] + + def test_computed_household_groups_require_people_counts(): household = _household_frame( [10, 20], diff --git a/uv.lock b/uv.lock index b12d83f5..c9a9b55e 100644 --- a/uv.lock +++ b/uv.lock @@ -2890,7 +2890,7 @@ requires-dist = [ { name = "itables", marker = "extra == 'dev'" }, { name = "jsonschema", specifier = ">=4.0.0" }, { name = "jupyter-book", marker = "extra == 'dev'" }, - { name = "microdf-python", specifier = ">=1.3.0" }, + { name = "microdf-python", specifier = "==1.3.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.11.0" }, { name = "networkx", marker = "extra == 'graph'", specifier = ">=3.0" }, { name = "packaging", specifier = ">=23.0" }, From 09f486d934e8e88ddcce7fee51abdb45166edf02 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Fri, 24 Jul 2026 19:04:38 +0300 Subject: [PATCH 4/5] Add decile impact behavior tests --- tests/test_decile_grouping.py | 149 ++++++ tests/test_intra_decile_impact.py | 728 +++++++++++++++++++++++++++++- 2 files changed, 856 insertions(+), 21 deletions(-) diff --git a/tests/test_decile_grouping.py b/tests/test_decile_grouping.py index f4001eb4..70c76e33 100644 --- a/tests/test_decile_grouping.py +++ b/tests/test_decile_grouping.py @@ -1,5 +1,6 @@ """Tests for shared weighted decile grouping.""" +import numpy as np import pandas as pd import pytest from microdf import MicroDataFrame @@ -46,6 +47,24 @@ def test_household_groups_multiply_survey_weights_by_household_size(): assert groups.tolist() == [6, 8, 9, 10] +def test_ten_equally_weighted_households_fill_all_ten_groups(): + household = _household_frame( + list(range(10, 110, 10)), + [1] * 10, + [1] * 10, + ) + + groups = calculate_decile_groups( + household, + household["household_net_income"], + decile_variable=None, + entity="household", + quantiles=10, + ) + + assert groups.tolist() == list(range(1, 11)) + + def test_household_groups_keep_tied_incomes_together(): household = _household_frame( [10, 10, 20], @@ -64,6 +83,24 @@ def test_household_groups_keep_tied_incomes_together(): assert groups.tolist() == [8, 8, 10] +def test_large_survey_observation_is_not_split_across_groups(): + household = _household_frame( + [10, 20], + [15, 85], + [1, 1], + ) + + groups = calculate_decile_groups( + household, + household["household_net_income"], + decile_variable=None, + entity="household", + quantiles=10, + ) + + assert groups.tolist() == [2, 10] + + def test_computed_groups_clamp_zero_weight_rows_to_first_group(): household = _household_frame( [10, 20, 30], @@ -82,6 +119,28 @@ def test_computed_groups_clamp_zero_weight_rows_to_first_group(): assert groups.tolist() == [1, 5, 10] +def test_non_household_groups_use_entity_survey_weight(): + person = MicroDataFrame( + pd.DataFrame( + { + "person_income": [10, 20], + "person_weight": [9, 1], + } + ), + weights="person_weight", + ) + + groups = calculate_decile_groups( + person, + person["person_income"], + decile_variable=None, + entity="person", + quantiles=10, + ) + + assert groups.tolist() == [9, 10] + + def test_computed_groups_exclude_negative_ranking_values(): household = _household_frame( [-10, 20, 30], @@ -100,6 +159,25 @@ def test_computed_groups_exclude_negative_ranking_values(): assert groups.tolist() == [-1, 7, 10] +def test_non_finite_ranking_values_do_not_affect_group_boundaries(): + household = _household_frame( + [10, 20, np.nan, np.inf, -np.inf], + [1, 1, 100, 100, 100], + [1, 1, 1, 1, 1], + ) + + groups = calculate_decile_groups( + household, + household["household_net_income"], + decile_variable=None, + entity="household", + quantiles=10, + ) + + assert groups.iloc[:2].tolist() == [5, 10] + assert all(pd.isna(group) or group < 1 or group > 10 for group in groups.iloc[2:]) + + def test_precomputed_groups_bypass_weighted_ranking_requirements(): household = _household_frame( [10, 20], @@ -150,3 +228,74 @@ def test_computed_household_groups_require_people_counts(): entity="household", quantiles=10, ) + + +@pytest.mark.parametrize("invalid_weight", [-1, np.nan, np.inf, -np.inf]) +def test_computed_groups_reject_invalid_survey_weights(invalid_weight): + household = _household_frame( + [10, 20], + [2, invalid_weight], + [1, 1], + ) + + with pytest.raises(ValueError): + calculate_decile_groups( + household, + household["household_net_income"], + decile_variable=None, + entity="household", + quantiles=10, + ) + + +@pytest.mark.parametrize("invalid_people_count", [-1, np.nan, np.inf, -np.inf]) +def test_computed_groups_reject_invalid_people_counts(invalid_people_count): + household = _household_frame( + [10, 20], + [1, 1], + [2, invalid_people_count], + ) + + with pytest.raises(ValueError): + calculate_decile_groups( + household, + household["household_net_income"], + decile_variable=None, + entity="household", + quantiles=10, + ) + + +def test_computed_groups_reject_zero_total_effective_weight(): + household = _household_frame( + [10, 20], + [0, 0], + [1, 1], + ) + + with pytest.raises((ValueError, ZeroDivisionError)): + calculate_decile_groups( + household, + household["household_net_income"], + decile_variable=None, + entity="household", + quantiles=10, + ) + + +@pytest.mark.parametrize("quantiles", [0, -1]) +def test_computed_groups_require_at_least_one_group(quantiles): + household = _household_frame( + [10, 20], + [1, 1], + [1, 1], + ) + + with pytest.raises(ValueError, match="quantiles"): + calculate_decile_groups( + household, + household["household_net_income"], + decile_variable=None, + entity="household", + quantiles=quantiles, + ) diff --git a/tests/test_intra_decile_impact.py b/tests/test_intra_decile_impact.py index 4b4cf08c..a51951d3 100644 --- a/tests/test_intra_decile_impact.py +++ b/tests/test_intra_decile_impact.py @@ -1,10 +1,11 @@ -"""Unit tests for IntraDecileImpact and DecileImpact with decile_variable.""" +"""Unit tests for DecileImpact and IntraDecileImpact.""" from typing import Optional from unittest.mock import MagicMock import numpy as np import pandas as pd +import pytest from microdf import MicroDataFrame from policyengine.core import Simulation, TaxBenefitModel, TaxBenefitModelVersion @@ -41,10 +42,15 @@ def _make_version(variable_name: str, entity: str) -> TaxBenefitModelVersion: return version -def _make_sim(household_data: dict, variables: Optional[list] = None) -> MagicMock: +def _make_sim( + household_data: dict, + variables: Optional[list] = None, + *, + index=None, +) -> MagicMock: """Create a mock Simulation with household-level data.""" hh_df = MicroDataFrame( - pd.DataFrame(household_data), + pd.DataFrame(household_data, index=index), weights="household_weight", ) sim = MagicMock() @@ -127,24 +133,22 @@ def test_intra_decile_all_large_gain(): assert r.no_change == 0.0 or abs(r.no_change) < 1e-9 -def test_intra_decile_overall_is_mean_of_deciles(): - """The overall row (decile=0) is the arithmetic mean of deciles 1-10.""" - n = 100 - incomes = np.linspace(10000, 100000, n) - # Give a small gain so results aren't trivially all in one bucket - reform_incomes = incomes * 1.02 # 2% gain (falls in gain_less_than_5pct) +def test_intra_decile_overall_uses_included_population_weights(): + """Overall proportions are calculated directly, not averaged by decile.""" baseline = _make_sim( { - "household_net_income": incomes, - "household_weight": np.ones(n), - "household_count_people": np.ones(n), + "household_net_income": [100.0, 100.0], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 9.0], + "household_income_decile": [1, 2], } ) reform = _make_sim( { - "household_net_income": reform_incomes, - "household_weight": np.ones(n), - "household_count_people": np.ones(n), + "household_net_income": [110.0, 100.0], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 9.0], + "household_income_decile": [1, 2], } ) @@ -152,19 +156,316 @@ def test_intra_decile_overall_is_mean_of_deciles(): baseline_simulation=baseline, reform_simulation=reform, income_variable="household_net_income", + decile_variable="household_income_decile", entity="household", + quantiles=2, ) - decile_rows = [r for r in results.outputs if r.decile != 0] overall = next(r for r in results.outputs if r.decile == 0) - assert len(decile_rows) == 10 + assert overall.gain_more_than_5pct == pytest.approx(0.1) + assert overall.no_change == pytest.approx(0.9) + assert overall.lose_more_than_5pct == pytest.approx(0.0) + + +def test_intra_decile_category_boundaries_are_inclusive_on_the_upper_bound(): + baseline = _make_sim( + { + "household_net_income": [1000.0] * 5, + "household_weight": [1.0] * 5, + "household_count_people": [1.0] * 5, + "household_income_decile": [1] * 5, + } + ) + reform = _make_sim( + { + "household_net_income": [950.0, 999.0, 1001.0, 1050.0, 1060.0], + "household_weight": [1.0] * 5, + "household_count_people": [1.0] * 5, + "household_income_decile": [1] * 5, + } + ) + + results = compute_intra_decile_impacts( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_income_decile", + entity="household", + quantiles=1, + ) + + group = next(result for result in results.outputs if result.decile == 1) + assert group.lose_more_than_5pct == pytest.approx(0.2) + assert group.lose_less_than_5pct == pytest.approx(0.2) + assert group.no_change == pytest.approx(0.2) + assert group.gain_less_than_5pct == pytest.approx(0.2) + assert group.gain_more_than_5pct == pytest.approx(0.2) + + +def test_intra_decile_uses_person_weights_within_each_group(): + baseline = _make_sim( + { + "household_net_income": [100.0, 100.0], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 9.0], + "household_income_decile": [1, 1], + } + ) + reform = _make_sim( + { + "household_net_income": [110.0, 100.0], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 9.0], + "household_income_decile": [1, 1], + } + ) + + results = compute_intra_decile_impacts( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_income_decile", + entity="household", + quantiles=1, + ) + + group = next(result for result in results.outputs if result.decile == 1) + assert group.gain_more_than_5pct == pytest.approx(0.1) + assert group.no_change == pytest.approx(0.9) + assert sum( + [ + group.lose_more_than_5pct, + group.lose_less_than_5pct, + group.no_change, + group.gain_less_than_5pct, + group.gain_more_than_5pct, + ] + ) == pytest.approx(1.0) + + +def test_intra_decile_percentage_change_floors_baseline_income_at_one(): + baseline = _make_sim( + { + "household_net_income": [0.0, -10.0], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 1.0], + "household_wealth_decile": [1, 1], + } + ) + reform = _make_sim( + { + "household_net_income": [0.04, -9.96], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 1.0], + "household_wealth_decile": [1, 1], + } + ) + + results = compute_intra_decile_impacts( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_wealth_decile", + entity="household", + quantiles=1, + ) + + group = next(result for result in results.outputs if result.decile == 1) + assert group.gain_less_than_5pct == pytest.approx(1.0) + + +def test_intra_decile_empty_groups_are_null(): + baseline = _make_sim( + { + "household_net_income": [100.0], + "household_weight": [1.0], + "household_count_people": [1.0], + "household_income_decile": [1], + } + ) + reform = _make_sim( + { + "household_net_income": [100.0], + "household_weight": [1.0], + "household_count_people": [1.0], + "household_income_decile": [1], + } + ) + + results = compute_intra_decile_impacts( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_income_decile", + entity="household", + quantiles=2, + ) + + empty_group = next(result for result in results.outputs if result.decile == 2) + assert empty_group.lose_more_than_5pct is None + assert empty_group.lose_less_than_5pct is None + assert empty_group.no_change is None + assert empty_group.gain_less_than_5pct is None + assert empty_group.gain_more_than_5pct is None + + +def test_intra_decile_overall_excludes_invalid_precomputed_groups(): + baseline = _make_sim( + { + "household_net_income": [100.0, 100.0, 100.0, 100.0], + "household_weight": [1.0, 99.0, 99.0, 99.0], + "household_count_people": [1.0, 1.0, 1.0, 1.0], + "household_income_decile": [1, -1, 0, 2], + } + ) + reform = _make_sim( + { + "household_net_income": [110.0, 0.0, 0.0, 0.0], + "household_weight": [1.0, 99.0, 99.0, 99.0], + "household_count_people": [1.0, 1.0, 1.0, 1.0], + "household_income_decile": [1, -1, 0, 2], + } + ) + + results = compute_intra_decile_impacts( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_income_decile", + entity="household", + quantiles=1, + ) + + overall = next(result for result in results.outputs if result.decile == 0) + assert overall.gain_more_than_5pct == pytest.approx(1.0) + assert overall.lose_more_than_5pct == pytest.approx(0.0) + + +def test_intra_decile_overall_is_null_without_included_positive_weight(): + baseline = _make_sim( + { + "household_net_income": [100.0, 200.0], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 1.0], + "household_income_decile": [-1, 3], + } + ) + reform = _make_sim( + { + "household_net_income": [110.0, 220.0], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 1.0], + "household_income_decile": [-1, 3], + } + ) + + results = compute_intra_decile_impacts( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_income_decile", + entity="household", + quantiles=2, + ) + + overall = next(result for result in results.outputs if result.decile == 0) + assert overall.lose_more_than_5pct is None + assert overall.lose_less_than_5pct is None + assert overall.no_change is None + assert overall.gain_less_than_5pct is None + assert overall.gain_more_than_5pct is None + + +def test_intra_decile_rejects_missing_reform_income_for_included_household(): + baseline = _make_sim( + { + "household_net_income": [100.0, 200.0], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 1.0], + "household_income_decile": [1, 1], + } + ) + reform = _make_sim( + { + "household_net_income": [110.0, np.nan], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 1.0], + "household_income_decile": [1, 1], + } + ) + + with pytest.raises(ValueError): + compute_intra_decile_impacts( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_income_decile", + entity="household", + quantiles=1, + ) + + +def test_intra_decile_allows_missing_reform_income_for_excluded_household(): + baseline = _make_sim( + { + "household_net_income": [100.0, 200.0], + "household_weight": [1.0, 100.0], + "household_count_people": [1.0, 1.0], + "household_income_decile": [1, -1], + } + ) + reform = _make_sim( + { + "household_net_income": [110.0, np.nan], + "household_weight": [1.0, 100.0], + "household_count_people": [1.0, 1.0], + "household_income_decile": [1, -1], + } + ) - expected_gain = sum(r.gain_less_than_5pct for r in decile_rows) / 10 - assert abs(overall.gain_less_than_5pct - expected_gain) < 1e-9 + results = compute_intra_decile_impacts( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_income_decile", + entity="household", + quantiles=1, + ) - expected_no_change = sum(r.no_change for r in decile_rows) / 10 - assert abs(overall.no_change - expected_no_change) < 1e-9 + overall = next(result for result in results.outputs if result.decile == 0) + assert overall.gain_more_than_5pct == pytest.approx(1.0) + + +def test_intra_decile_rejects_misaligned_simulation_observations(): + baseline = _make_sim( + { + "household_net_income": [100.0, 200.0], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 1.0], + "household_income_decile": [1, 1], + }, + index=["a", "b"], + ) + reform = _make_sim( + { + "household_net_income": [110.0, 220.0], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 1.0], + "household_income_decile": [1, 1], + }, + index=["a", "c"], + ) + + with pytest.raises(ValueError): + compute_intra_decile_impacts( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_income_decile", + entity="household", + quantiles=1, + ) def test_intra_decile_with_decile_variable(): @@ -261,6 +562,391 @@ def test_decile_impact_with_decile_variable(): assert abs(di.absolute_change - 2000.0) < 1e-6 +def test_decile_impact_uses_household_weights_after_person_weighted_grouping(): + """Group statistics describe households, not people in those households.""" + variables = [_make_variable_mock("household_net_income", "household")] + baseline = _make_sim( + { + "household_net_income": [10.0, 30.0], + "household_weight": [1.0, 3.0], + "household_count_people": [100.0, 1.0], + "household_income_decile": [1, 1], + }, + variables=variables, + ) + reform = _make_sim( + { + "household_net_income": [20.0, 50.0], + "household_weight": [1.0, 3.0], + "household_count_people": [100.0, 1.0], + "household_income_decile": [1, 1], + }, + variables=variables, + ) + + impact = DecileImpact.model_construct( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_income_decile", + entity="household", + decile=1, + quantiles=1, + ) + impact.run() + + assert impact.baseline_mean == pytest.approx(25.0) + assert impact.reform_mean == pytest.approx(42.5) + assert impact.absolute_change == pytest.approx(17.5) + assert impact.relative_change == pytest.approx(70.0) + assert impact.count_better_off == pytest.approx(4.0) + assert impact.count_worse_off == pytest.approx(0.0) + assert impact.count_no_change == pytest.approx(0.0) + + +def test_decile_impact_counts_better_worse_and_no_change_separately(): + variables = [_make_variable_mock("household_net_income", "household")] + baseline = _make_sim( + { + "household_net_income": [10.0, 20.0, 30.0], + "household_weight": [2.0, 3.0, 5.0], + "household_count_people": [1.0, 1.0, 1.0], + "household_income_decile": [1, 1, 1], + }, + variables=variables, + ) + reform = _make_sim( + { + "household_net_income": [11.0, 19.0, 30.0], + "household_weight": [2.0, 3.0, 5.0], + "household_count_people": [1.0, 1.0, 1.0], + "household_income_decile": [1, 1, 1], + }, + variables=variables, + ) + + impact = DecileImpact.model_construct( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_income_decile", + entity="household", + decile=1, + quantiles=1, + ) + impact.run() + + assert impact.count_better_off == pytest.approx(2.0) + assert impact.count_worse_off == pytest.approx(3.0) + assert impact.count_no_change == pytest.approx(5.0) + + +def test_decile_impact_relative_change_is_null_for_zero_group_baseline(): + """Negative measured income remains included in a valid wealth group.""" + variables = [_make_variable_mock("household_net_income", "household")] + baseline = _make_sim( + { + "household_net_income": [-10.0, 10.0], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 1.0], + "household_wealth_decile": [1, 1], + }, + variables=variables, + ) + reform = _make_sim( + { + "household_net_income": [-5.0, 15.0], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 1.0], + "household_wealth_decile": [1, 1], + }, + variables=variables, + ) + + impact = DecileImpact.model_construct( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_wealth_decile", + entity="household", + decile=1, + quantiles=1, + ) + impact.run() + + assert impact.baseline_mean == pytest.approx(0.0) + assert impact.reform_mean == pytest.approx(5.0) + assert impact.absolute_change == pytest.approx(5.0) + assert impact.relative_change is None + assert impact.count_better_off == pytest.approx(2.0) + + +def test_decile_impact_empty_group_has_null_statistics_and_zero_counts(): + variables = [_make_variable_mock("household_net_income", "household")] + baseline = _make_sim( + { + "household_net_income": [100.0], + "household_weight": [1.0], + "household_count_people": [1.0], + "household_income_decile": [1], + }, + variables=variables, + ) + reform = _make_sim( + { + "household_net_income": [110.0], + "household_weight": [1.0], + "household_count_people": [1.0], + "household_income_decile": [1], + }, + variables=variables, + ) + + impact = DecileImpact.model_construct( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_income_decile", + entity="household", + decile=2, + quantiles=2, + ) + impact.run() + + assert impact.baseline_mean is None + assert impact.reform_mean is None + assert impact.absolute_change is None + assert impact.relative_change is None + assert impact.count_better_off == pytest.approx(0.0) + assert impact.count_worse_off == pytest.approx(0.0) + assert impact.count_no_change == pytest.approx(0.0) + + +def test_decile_impact_zero_weight_household_does_not_affect_results(): + variables = [_make_variable_mock("household_net_income", "household")] + baseline = _make_sim( + { + "household_net_income": [10.0, 1000.0], + "household_weight": [1.0, 0.0], + "household_count_people": [1.0, 1.0], + "household_income_decile": [1, 1], + }, + variables=variables, + ) + reform = _make_sim( + { + "household_net_income": [20.0, 0.0], + "household_weight": [1.0, 0.0], + "household_count_people": [1.0, 1.0], + "household_income_decile": [1, 1], + }, + variables=variables, + ) + + impact = DecileImpact.model_construct( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_income_decile", + entity="household", + decile=1, + quantiles=1, + ) + impact.run() + + assert impact.baseline_mean == pytest.approx(10.0) + assert impact.reform_mean == pytest.approx(20.0) + assert impact.absolute_change == pytest.approx(10.0) + assert impact.relative_change == pytest.approx(100.0) + assert impact.count_better_off == pytest.approx(1.0) + assert impact.count_worse_off == pytest.approx(0.0) + + +def test_decile_impact_uses_baseline_group_membership(): + variables = [_make_variable_mock("household_net_income", "household")] + baseline = _make_sim( + { + "household_net_income": [10.0, 20.0], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 1.0], + }, + variables=variables, + ) + reform = _make_sim( + { + "household_net_income": [1000.0, -1000.0], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 1.0], + }, + variables=variables, + ) + + impact = DecileImpact.model_construct( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + entity="household", + decile=1, + quantiles=2, + ) + impact.run() + + assert impact.baseline_mean == pytest.approx(10.0) + assert impact.reform_mean == pytest.approx(1000.0) + + +def test_decile_impact_excludes_invalid_precomputed_groups(): + variables = [_make_variable_mock("household_net_income", "household")] + baseline = _make_sim( + { + "household_net_income": [10.0, 20.0, 1000.0, 1000.0, 1000.0, 1000.0], + "household_weight": [1.0] * 6, + "household_count_people": [1.0] * 6, + "household_income_decile": [1, 2, -1, 0, 3, np.nan], + }, + variables=variables, + ) + reform = _make_sim( + { + "household_net_income": [11.0, 22.0, 0.0, 0.0, 0.0, 0.0], + "household_weight": [1.0] * 6, + "household_count_people": [1.0] * 6, + "household_income_decile": [1, 2, -1, 0, 3, np.nan], + }, + variables=variables, + ) + + impacts = [] + for decile in (1, 2): + impact = DecileImpact.model_construct( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_income_decile", + entity="household", + decile=decile, + quantiles=2, + ) + impact.run() + impacts.append(impact) + + assert impacts[0].baseline_mean == pytest.approx(10.0) + assert impacts[0].absolute_change == pytest.approx(1.0) + assert impacts[0].count_better_off == pytest.approx(1.0) + assert impacts[1].baseline_mean == pytest.approx(20.0) + assert impacts[1].absolute_change == pytest.approx(2.0) + assert impacts[1].count_better_off == pytest.approx(1.0) + + +def test_decile_impact_rejects_missing_reform_income_for_included_household(): + variables = [_make_variable_mock("household_net_income", "household")] + baseline = _make_sim( + { + "household_net_income": [10.0, 20.0], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 1.0], + "household_income_decile": [1, 1], + }, + variables=variables, + ) + reform = _make_sim( + { + "household_net_income": [11.0, np.nan], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 1.0], + "household_income_decile": [1, 1], + }, + variables=variables, + ) + + impact = DecileImpact.model_construct( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_income_decile", + entity="household", + decile=1, + quantiles=1, + ) + + with pytest.raises(ValueError): + impact.run() + + +def test_decile_impact_allows_missing_reform_income_for_excluded_household(): + variables = [_make_variable_mock("household_net_income", "household")] + baseline = _make_sim( + { + "household_net_income": [10.0, 20.0], + "household_weight": [1.0, 100.0], + "household_count_people": [1.0, 1.0], + "household_income_decile": [1, -1], + }, + variables=variables, + ) + reform = _make_sim( + { + "household_net_income": [11.0, np.nan], + "household_weight": [1.0, 100.0], + "household_count_people": [1.0, 1.0], + "household_income_decile": [1, -1], + }, + variables=variables, + ) + + impact = DecileImpact.model_construct( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_income_decile", + entity="household", + decile=1, + quantiles=1, + ) + impact.run() + + assert impact.baseline_mean == pytest.approx(10.0) + assert impact.reform_mean == pytest.approx(11.0) + assert impact.absolute_change == pytest.approx(1.0) + + +def test_decile_impact_rejects_misaligned_simulation_observations(): + variables = [_make_variable_mock("household_net_income", "household")] + baseline = _make_sim( + { + "household_net_income": [10.0, 20.0], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 1.0], + "household_income_decile": [1, 1], + }, + variables=variables, + index=["a", "b"], + ) + reform = _make_sim( + { + "household_net_income": [11.0, 21.0], + "household_weight": [1.0, 1.0], + "household_count_people": [1.0, 1.0], + "household_income_decile": [1, 1], + }, + variables=variables, + index=["a", "c"], + ) + + impact = DecileImpact.model_construct( + baseline_simulation=baseline, + reform_simulation=reform, + income_variable="household_net_income", + decile_variable="household_income_decile", + entity="household", + decile=1, + quantiles=1, + ) + + with pytest.raises(ValueError): + impact.run() + + def test_calculate_decile_impacts_with_decile_variable(monkeypatch): """calculate_decile_impacts passes pre-computed grouping through.""" version = _make_version("household_net_income", "household") From c295f6513d97ab84ed79ed7914541782e93dc047 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Fri, 24 Jul 2026 20:09:26 +0300 Subject: [PATCH 5/5] Implement weighted decile impact semantics --- changelog.d/485.breaking.md | 2 +- docs/decile-impact-spec.md | 298 ++++++++++++++++++ docs/impact-analysis.md | 2 +- docs/outputs.md | 12 +- src/policyengine/outputs/decile_analysis.py | 178 +++++++++++ src/policyengine/outputs/decile_grouping.py | 126 ++++++-- src/policyengine/outputs/decile_impact.py | 158 ++++++---- .../outputs/intra_decile_impact.py | 158 ++++++---- 8 files changed, 783 insertions(+), 151 deletions(-) create mode 100644 docs/decile-impact-spec.md create mode 100644 src/policyengine/outputs/decile_analysis.py diff --git a/changelog.d/485.breaking.md b/changelog.d/485.breaking.md index 7428fcd8..6127e972 100644 --- a/changelog.d/485.breaking.md +++ b/changelog.d/485.breaking.md @@ -1 +1 @@ -Changed default decile-impact calculations to household net income and person-weighted household groups, excluded negative computed-income values from reported deciles, and preserved explicit selection of equivalised HBAI net income. +Changed default decile-impact calculations to household net income and person-weighted household groups, excluded invalid computed-income values from reported deciles, preserved explicit selection of equivalised HBAI net income, made relative changes compare weighted group means, and calculated overall intra-decile results directly from the included population. diff --git a/docs/decile-impact-spec.md b/docs/decile-impact-spec.md new file mode 100644 index 00000000..2f73a40b --- /dev/null +++ b/docs/decile-impact-spec.md @@ -0,0 +1,298 @@ +# Decile impact behavior specification + +Status: Approved and implemented. + +## Scope + +This specification covers: + +- computed income groups; +- precomputed grouping variables; +- `DecileImpact` results; +- `IntraDecileImpact` results; +- the overall intra-decile result; and +- invalid, missing, zero-weight, tied, and excluded observations. + +Unless configured otherwise, calculations use ten groups. + +## Definitions + +`income_variable` +: The income concept being measured. When no `decile_variable` is supplied, it + is also the value used to construct groups. + +`decile_variable` +: An optional precomputed grouping variable. When present, this variable + determines group membership while `income_variable` remains the measured + outcome. + +`effective grouping weight` +: For households, `household_weight * household_count_people`. For another + entity, that entity's survey weight. + +`analysis weight` +: The weight used to calculate a reported statistic after groups have been + constructed. `DecileImpact` uses the entity survey weight. + `IntraDecileImpact` uses the effective grouping weight. + +`excluded observation` +: An observation that does not belong to any reported group. It does not + contribute to per-group or overall reported results. + +## Group construction + +Groups are always based on the baseline simulation. Reform values never cause +an observation to move between groups. + +### Computed groups + +When no `decile_variable` is supplied: + +1. Rank observations by the baseline value of `income_variable`. +2. Use the effective grouping weight to calculate cumulative population shares. +3. Keep observations with equal ranking values in the same group. +4. Assign each observation to: + + ```text + ceil(cumulative population share * number of groups) + ``` + +5. Clamp the result to the inclusive range from 1 through the number of groups. + +Each survey observation belongs to no more than one group. An observation is +never fractionally split across group boundaries. + +Because observations and tied values are indivisible, reported groups are not +guaranteed to contain equal populations. Groups may be unequal or empty. + +### Precomputed groups + +When `decile_variable` is supplied: + +- use its baseline value directly; +- include values from 1 through the configured number of groups; +- exclude missing values and values outside that range, including the + conventional `-1` sentinel; and +- do not recalculate or rebalance the supplied groups. + +This preserves country-package sentinels such as `-1` for negative income or +wealth. + +## Exclusions and invalid inputs + +| Case | Result | +|---|---| +| Negative computed ranking income | Include its weight when determining percentile positions, then assign group `-1` and exclude it from reported results | +| Missing or non-finite computed ranking income | Exclude it from ranking and reported results; it does not affect group boundaries | +| Zero effective grouping weight | Permit the row and clamp any computed group to at least 1. It has no influence on group construction or effective-weight results; it can still contribute to an entity-survey-weighted `DecileImpact` when its survey weight is positive | +| Negative or non-finite survey weight | Raise an error | +| All effective grouping weights are zero | Raise an error | +| Missing `household_count_people` for computed household groups | Raise an error | +| Negative or non-finite `household_count_people` | Raise an error | +| Number of groups less than one | Raise an error | +| Missing reform income for an included observation | Raise an error | +| Baseline and reform observations do not align | Raise an error | + +## Worked grouping examples + +### Ten equally weighted observations + +```text +Income: 10 20 30 40 50 60 70 80 90 100 +Weight: 1 1 1 1 1 1 1 1 1 1 +Group: 1 2 3 4 5 6 7 8 9 10 +``` + +### Tied observations + +```text +Income: 10 10 20 +Effective weight: 1 2 1 +Group: 8 8 10 +``` + +The observations with income 10 jointly represent 75% of the population. +Max-rank semantics therefore place both in group 8. Groups 1 through 7 and +group 9 are empty. + +### One unusually large survey observation + +```text +Income: 10 20 +Effective weight: 15 85 +Group: 2 10 +``` + +The first observation represents 15% of the population and belongs wholly to +group 2. It is not divided between groups 1 and 2. + +### Leading zero-weight observation + +```text +Income: 10 20 30 +Effective weight: 0 1 1 +Group: 1 5 10 +``` + +The first observation is lower-clamped to group 1 but contributes nothing to +weighted statistics. + +### Negative income + +```text +Income: -10 20 30 +Effective weight: 1 1 1 +Ranked group: 4 7 10 +Reported group: -1 7 10 +``` + +The negative-income observation affects the percentile positions, matching the +country-package convention, but is excluded from reported group results. + +## `DecileImpact` + +### Populated groups + +For each populated group: + +`baseline_mean` +: Survey-weighted mean baseline income. + +`reform_mean` +: Survey-weighted mean reform income. + +`absolute_change` +: `reform_mean - baseline_mean`. + +`relative_change` +: Calculated as: + + ```text + 100 * absolute_change / baseline_mean + ``` + + If `baseline_mean` is zero, the result is null. + +`count_better_off` +: Survey-weighted count of observations whose reform income is greater than + baseline income. + +`count_worse_off` +: Survey-weighted count of observations whose reform income is less than + baseline income. + +`count_no_change` +: Survey-weighted count of observations whose reform and baseline incomes are + equal. + +Household group boundaries are person-weighted, but the means and counts remain +household-survey-weighted. Person weights determine the groups; the statistics +describe households within those groups. + +### Empty groups + +An empty group still produces a result row: + +```text +baseline_mean: null +reform_mean: null +absolute_change: null +relative_change: null +count_better_off: 0 +count_worse_off: 0 +count_no_change: 0 +``` + +An empty group must not be represented as a populated group in which everyone +experienced no change. + +## `IntraDecileImpact` + +For every included household, calculate: + +```text +(reform income - baseline income) / max(baseline income, 1) +``` + +Classify the result into exactly one category: + +| Category | Definition | +|---|---| +| `lose_more_than_5pct` | Change less than or equal to -5% | +| `lose_less_than_5pct` | Change greater than -5% and less than or equal to -0.1% | +| `no_change` | Change greater than -0.1% and less than or equal to 0.1% | +| `gain_less_than_5pct` | Change greater than 0.1% and less than or equal to 5% | +| `gain_more_than_5pct` | Change greater than 5% | + +Category proportions use: + +```text +household_weight * household_count_people +``` + +For a populated group, the five category proportions must sum to 1, subject +only to floating-point tolerance. + +For an empty group, all five category proportions are null. + +## Overall intra-decile result + +The `decile=0` result is calculated directly from all included observations: + +```text +overall category proportion = + effective weight in the category + / total included effective weight +``` + +It is not the arithmetic mean of the individual group proportions. + +This remains correct when: + +- groups are unequal; +- groups are empty; +- tied observations skip group labels; +- one observation has an unusually large weight; or +- precomputed groups exclude observations. + +If no included observation has positive effective weight, all overall +proportions are null. + +## Grouping versus the measured outcome + +Without `decile_variable`, `income_variable` controls both group construction +and the measured outcome. + +With `decile_variable`: + +- `decile_variable` controls group membership; and +- `income_variable` controls the measured outcome. + +For example: + +```python +income_variable = "household_net_income" +decile_variable = "household_wealth_decile" +``` + +This measures household-net-income changes within baseline wealth groups. + +Exclusion is based on the grouping concept rather than the measured outcome. A +negative measured income remains included when its precomputed group is valid. +Therefore: + +- a negative income used to construct income groups is excluded; +- negative wealth represented by wealth group `-1` is excluded; and +- negative household income inside a valid wealth group remains included. + +## Approved decisions + +1. Tied values remain together even when this creates unequal or empty groups. +2. Negative computed ranking income affects group boundaries but is excluded + from reported results afterward. +3. Negative measured income inside a valid non-income group remains included. +4. Empty-group statistics are null, with winner/loser/no-change counts of zero. +5. The overall intra-decile result is calculated directly from the included + population. +6. `relative_change` is the percentage change between the reported group means, + rather than the mean of observation-level percentage changes. diff --git a/docs/impact-analysis.md b/docs/impact-analysis.md index 9063db68..cdcd2ad8 100644 --- a/docs/impact-analysis.md +++ b/docs/impact-analysis.md @@ -31,7 +31,7 @@ A `PolicyReformAnalysis` with: | Attribute | Type | Content | |---|---|---| -| `decile_impacts` | `OutputCollection[DecileImpact]` | Household net income baseline / reform / change and winner-loser counts per person-weighted income decile; negative-income rows are excluded from reported deciles | +| `decile_impacts` | `OutputCollection[DecileImpact]` | Household-weighted net income means, changes, and outcome counts within person-weighted income deciles; negative-income rows are excluded from reported deciles | | `wealth_decile_impacts` | `OutputCollection[DecileImpact]` | UK only: household net income impacts grouped by `household_wealth_decile` | | `intra_wealth_decile_impacts` | `OutputCollection[IntraDecileImpact]` | UK only: within-wealth-decile distribution of household net income changes | | `program_statistics` | `OutputCollection[ProgramStatistics]` | Totals, counts, winners/losers per program | diff --git a/docs/outputs.md b/docs/outputs.md index 15ca1f27..2114aff4 100644 --- a/docs/outputs.md +++ b/docs/outputs.md @@ -96,7 +96,11 @@ Or on a relative change — `relative_change_geq=0.05` selects households with a One decile's baseline mean, reform mean, and mean change. For all ten at once, use `calculate_decile_impacts`. -By default, `calculate_decile_impacts` measures `household_net_income` and ranks units into deciles using that variable. Household groups are person-weighted: their survey weights are multiplied by `household_count_people` before ranking. Rows with negative values of the computed income concept receive the conventional `-1` group and are excluded from the reported decile results. Pass another `income_variable`, such as `equiv_hbai_household_net_income`, to select a different income concept explicitly. To measure changes in one variable while grouping by an existing decile variable, pass `decile_variable`. Precomputed group values outside `1..quantiles` are excluded, which preserves country-package sentinel values such as `-1` for negative income or wealth. For example, UK wealth-decile impacts measure changes in household net income grouped by `household_wealth_decile`. +By default, `calculate_decile_impacts` measures `household_net_income` and ranks units into deciles using that variable. Household groups are person-weighted: their survey weights are multiplied by `household_count_people` before ranking. Rows with negative values of the computed income concept receive the conventional `-1` group and are excluded from the reported decile results. Missing and non-finite ranking values are also excluded and do not affect group boundaries. + +The statistics reported after grouping use the entity survey weight. For household analysis, this means the decile boundaries are person-weighted while baseline and reform means and better-off, worse-off, and no-change counts are household-weighted. `relative_change` is the percentage change between the two reported group means, not the mean of household-level percentage changes. Empty groups have null means and changes and zero outcome counts. + +Pass another `income_variable`, such as `equiv_hbai_household_net_income`, to select a different income concept explicitly. To measure changes in one variable while grouping by an existing decile variable, pass `decile_variable`. Precomputed group values outside `1..quantiles` are excluded, which preserves country-package sentinel values such as `-1` for negative income or wealth. For example, UK wealth-decile impacts measure changes in household net income grouped by `household_wealth_decile`. ```python from policyengine.outputs import calculate_decile_impacts @@ -123,7 +127,11 @@ impacts.dataframe # includes the decile_variable column ## IntraDecileImpact -Distribution of household-level impact within each decile (five bucket categories summing to 1.0). Use `compute_intra_decile_impacts` for the full set. Its computed household groups use the same person-weighted ranking as `calculate_decile_impacts`, and its category proportions are also person-weighted. This helper accepts `decile_variable` when the grouping variable is already present in the simulation output. +Distribution of household-level impact within each decile (five bucket categories summing to 1.0 for populated groups). Use `compute_intra_decile_impacts` for the full set. Its computed household groups use the same person-weighted ranking as `calculate_decile_impacts`, and its category proportions are also person-weighted. Empty groups have null proportions. + +The `decile=0` row is calculated directly from all included people. It is not the arithmetic mean of the individual decile proportions, so it remains valid when tied values, large survey observations, exclusions, or empty groups make decile populations unequal. + +This helper accepts `decile_variable` when the grouping variable is already present in the simulation output. ```python from policyengine.outputs import compute_intra_decile_impacts diff --git a/src/policyengine/outputs/decile_analysis.py b/src/policyengine/outputs/decile_analysis.py new file mode 100644 index 00000000..0e9ff766 --- /dev/null +++ b/src/policyengine/outputs/decile_analysis.py @@ -0,0 +1,178 @@ +"""Shared preparation for decile-based baseline-reform analysis.""" + +from dataclasses import dataclass +from typing import Optional + +import numpy as np +import pandas as pd + +from policyengine.core import Simulation +from policyengine.outputs.decile_grouping import ( + _get_analysis_weight, + _get_decile_weights, + calculate_decile_groups, +) + + +@dataclass(frozen=True) +class _PreparedDecileAnalysis: + """Validated arrays and masks shared by decile impact outputs.""" + + entity: str + quantiles: int + index: pd.Index + baseline_income: np.ndarray + reform_income: np.ndarray + groups: pd.Series + analysis_weight: np.ndarray + effective_weight: np.ndarray + included: np.ndarray + + +def _variable_entity( + simulation: Simulation, + variable_name: str, +) -> Optional[str]: + """Return a variable's entity when model metadata is available.""" + variables = getattr( + simulation.tax_benefit_model_version, + "variables", + (), + ) + try: + variable = next( + variable for variable in variables if variable.name == variable_name + ) + except (StopIteration, TypeError): + return None + return str(variable.entity) + + +def _income_at_entity( + simulation: Simulation, + *, + income_variable: str, + variable_entity: Optional[str], + target_entity: str, +): + """Read or map an income variable onto the target analysis entity.""" + output_dataset = simulation.output_dataset + if output_dataset is None: + raise ValueError("Simulation output dataset is not available") + data = output_dataset.data + if data is None: + raise ValueError("Simulation output data is not available") + if variable_entity is not None and variable_entity != target_entity: + mapped = data.map_to_entity(variable_entity, target_entity) + return mapped[income_variable] + return getattr(data, target_entity)[income_variable] + + +def _prepare_decile_analysis( + baseline_simulation: Simulation, + reform_simulation: Simulation, + *, + income_variable: str, + decile_variable: Optional[str], + entity: Optional[str], + quantiles: int, + require_effective_weight: bool = False, +) -> _PreparedDecileAnalysis: + """Validate and prepare inputs used by both decile impact outputs.""" + if quantiles < 1: + raise ValueError("quantiles must be at least 1") + + variable_entity = _variable_entity( + baseline_simulation, + income_variable, + ) + target_entity = entity or variable_entity + if target_entity is None: + raise ValueError( + f"Could not determine the entity for income variable '{income_variable}'" + ) + + baseline_output = baseline_simulation.output_dataset + reform_output = reform_simulation.output_dataset + if baseline_output is None or baseline_output.data is None: + raise ValueError("Baseline simulation output data is not available") + if reform_output is None or reform_output.data is None: + raise ValueError("Reform simulation output data is not available") + baseline_data = getattr(baseline_output.data, target_entity) + reform_data = getattr(reform_output.data, target_entity) + if not baseline_data.index.equals(reform_data.index): + raise ValueError( + "Baseline and reform simulation observations must have identical indexes" + ) + + baseline_income_series = _income_at_entity( + baseline_simulation, + income_variable=income_variable, + variable_entity=variable_entity, + target_entity=target_entity, + ) + reform_income_series = _income_at_entity( + reform_simulation, + income_variable=income_variable, + variable_entity=variable_entity, + target_entity=target_entity, + ) + if not baseline_income_series.index.equals(baseline_data.index): + raise ValueError("Baseline income values must align with baseline observations") + if not reform_income_series.index.equals(reform_data.index): + raise ValueError("Reform income values must align with reform observations") + + baseline_income = np.asarray(baseline_income_series, dtype=float) + reform_income = np.asarray(reform_income_series, dtype=float) + analysis_weight = _get_analysis_weight( + baseline_data, + entity=target_entity, + ) + if require_effective_weight or decile_variable is None: + _, effective_weight = _get_decile_weights( + baseline_data, + entity=target_entity, + ) + else: + # A precomputed group plus a household-weighted output does not need + # household size. Keep the prepared shape uniform without imposing an + # unrelated input requirement. + effective_weight = analysis_weight.copy() + groups = calculate_decile_groups( + baseline_data, + baseline_income_series, + decile_variable=decile_variable, + entity=target_entity, + quantiles=quantiles, + ) + included = groups.isin(range(1, quantiles + 1)).to_numpy(dtype=bool) + + if not np.all(np.isfinite(baseline_income[included])): + raise ValueError( + "Included observations must have finite baseline income values" + ) + if not np.all(np.isfinite(reform_income[included])): + raise ValueError("Included observations must have finite reform income values") + + return _PreparedDecileAnalysis( + entity=target_entity, + quantiles=quantiles, + index=baseline_data.index, + baseline_income=baseline_income, + reform_income=reform_income, + groups=groups, + analysis_weight=analysis_weight, + effective_weight=effective_weight, + included=included, + ) + + +def _weighted_mean( + values: np.ndarray, + weights: np.ndarray, +) -> Optional[float]: + """Return a weighted mean, or ``None`` without positive total weight.""" + total_weight = float(np.sum(weights)) + if total_weight == 0: + return None + return float(np.sum(values * weights) / total_weight) diff --git a/src/policyengine/outputs/decile_grouping.py b/src/policyengine/outputs/decile_grouping.py index 9e462637..c7943eaf 100644 --- a/src/policyengine/outputs/decile_grouping.py +++ b/src/policyengine/outputs/decile_grouping.py @@ -7,6 +7,69 @@ from microdf import MicroSeries +def _validate_nonnegative_finite(values: np.ndarray, *, name: str) -> None: + """Require finite, non-negative values used as survey weights.""" + if not np.all(np.isfinite(values)): + raise ValueError(f"{name} must contain only finite values") + if np.any(values < 0): + raise ValueError(f"{name} must not contain negative values") + + +def _get_analysis_weight( + baseline_data: Any, + *, + entity: str, +) -> np.ndarray: + """Return validated entity survey weights.""" + weight_variable = f"{entity}_weight" + if weight_variable not in baseline_data.columns: + raise ValueError( + f"Weighted quantile grouping requires '{weight_variable}' in " + f"baseline output data for entity '{entity}'." + ) + + analysis_weight = np.asarray( + baseline_data[weight_variable], + dtype=float, + ) + _validate_nonnegative_finite( + analysis_weight, + name=weight_variable, + ) + return analysis_weight + + +def _get_decile_weights( + baseline_data: Any, + *, + entity: str, +) -> tuple[np.ndarray, np.ndarray]: + """Return entity survey weights and effective population weights.""" + analysis_weight = _get_analysis_weight( + baseline_data, + entity=entity, + ) + if entity != "household": + return analysis_weight, analysis_weight.copy() + + multiplier_variable = "household_count_people" + if multiplier_variable not in baseline_data.columns: + raise ValueError( + "Person-weighted household quantile grouping requires " + "'household_count_people' in baseline output data." + ) + + people_count = np.asarray( + baseline_data[multiplier_variable], + dtype=float, + ) + _validate_nonnegative_finite( + people_count, + name=multiplier_variable, + ) + return analysis_weight, analysis_weight * people_count + + def calculate_decile_groups( baseline_data: Any, ranking_values: Any, @@ -14,7 +77,7 @@ def calculate_decile_groups( decile_variable: Optional[str], entity: str, quantiles: int, -) -> Any: +) -> pd.Series: """Return precomputed groups or weighted ranks of ``ranking_values``. Household income groups follow the convention used by both country @@ -29,42 +92,49 @@ def calculate_decile_groups( intentionally exclude rows from reported groups. """ - if decile_variable: - return baseline_data[decile_variable] if quantiles < 1: raise ValueError("quantiles must be at least 1") + if decile_variable: + return pd.Series( + np.asarray(baseline_data[decile_variable]), + index=baseline_data.index, + ) - weight_variable = f"{entity}_weight" - if weight_variable not in baseline_data.columns: + _, effective_weight = _get_decile_weights( + baseline_data, + entity=entity, + ) + if np.sum(effective_weight) == 0: + raise ValueError("Effective grouping weights must have a positive total") + + ranking_array = np.asarray(ranking_values, dtype=float) + if len(ranking_array) != len(baseline_data): + raise ValueError("Ranking values must align with baseline output data") + + finite = np.isfinite(ranking_array) + groups = pd.Series( + pd.array([pd.NA] * len(ranking_array), dtype="Int64"), + index=baseline_data.index, + ) + if not np.any(finite): + return groups + + finite_weights = effective_weight[finite] + if np.sum(finite_weights) == 0: raise ValueError( - f"Weighted quantile grouping requires '{weight_variable}' in " - f"baseline output data for entity '{entity}'." - ) - weights = np.asarray(baseline_data[weight_variable], dtype=float) - - if entity == "household": - multiplier_variable = "household_count_people" - if multiplier_variable not in baseline_data.columns: - raise ValueError( - "Person-weighted household quantile grouping requires " - "'household_count_people' in baseline output data." - ) - weights = weights * np.asarray( - baseline_data[multiplier_variable], - dtype=float, + "Finite ranking values must have a positive effective weight total" ) - - ranking_array = np.asarray(ranking_values) weighted_values = MicroSeries( - ranking_array, - index=baseline_data.index, - weights=weights, + ranking_array[finite], + index=baseline_data.index[finite], + weights=finite_weights, ) percentile_ranks = np.asarray(weighted_values.rank(pct=True)) - groups = np.clip( + finite_groups = np.clip( np.ceil(percentile_ranks * quantiles), 1, quantiles, ).astype(int) - groups[ranking_array < 0] = -1 - return pd.Series(groups, index=baseline_data.index) + finite_groups[ranking_array[finite] < 0] = -1 + groups.loc[finite] = finite_groups + return groups diff --git a/src/policyengine/outputs/decile_impact.py b/src/policyengine/outputs/decile_impact.py index 02ba0ab7..75f4494a 100644 --- a/src/policyengine/outputs/decile_impact.py +++ b/src/policyengine/outputs/decile_impact.py @@ -1,5 +1,7 @@ +from dataclasses import dataclass from typing import Optional +import numpy as np import pandas as pd from pydantic import ConfigDict @@ -8,7 +10,71 @@ from policyengine.core.dynamic import Dynamic from policyengine.core.policy import Policy from policyengine.core.tax_benefit_model_version import TaxBenefitModelVersion -from policyengine.outputs.decile_grouping import calculate_decile_groups +from policyengine.outputs.decile_analysis import ( + _prepare_decile_analysis, + _PreparedDecileAnalysis, + _weighted_mean, +) + + +@dataclass(frozen=True) +class _DecileImpactValues: + """Calculated values for one reported decile.""" + + baseline_mean: Optional[float] + reform_mean: Optional[float] + absolute_change: Optional[float] + relative_change: Optional[float] + count_better_off: float + count_worse_off: float + count_no_change: float + + +def _calculate_decile_impact_values( + analysis: _PreparedDecileAnalysis, + *, + decile: int, +) -> _DecileImpactValues: + """Calculate household-weighted statistics for one decile.""" + in_decile = analysis.included & analysis.groups.eq(decile).fillna(False).to_numpy( + dtype=bool + ) + weights = analysis.analysis_weight[in_decile] + baseline_mean = _weighted_mean( + analysis.baseline_income[in_decile], + weights, + ) + reform_mean = _weighted_mean( + analysis.reform_income[in_decile], + weights, + ) + if baseline_mean is None or reform_mean is None: + return _DecileImpactValues( + baseline_mean=None, + reform_mean=None, + absolute_change=None, + relative_change=None, + count_better_off=0.0, + count_worse_off=0.0, + count_no_change=0.0, + ) + + income_change = ( + analysis.reform_income[in_decile] - analysis.baseline_income[in_decile] + ) + absolute_change = reform_mean - baseline_mean + relative_change = ( + None if baseline_mean == 0 else float(100 * absolute_change / baseline_mean) + ) + return _DecileImpactValues( + baseline_mean=baseline_mean, + reform_mean=reform_mean, + absolute_change=absolute_change, + relative_change=relative_change, + count_better_off=float(np.sum(weights[income_change > 0])), + count_worse_off=float(np.sum(weights[income_change < 0])), + count_no_change=float(np.sum(weights[income_change == 0])), + ) class DecileImpact(Output): @@ -33,64 +99,34 @@ class DecileImpact(Output): count_worse_off: Optional[float] = None count_no_change: Optional[float] = None - def run(self): + def run(self) -> None: """Calculate impact for this specific decile.""" - # Get variable object to determine entity - var_obj = next( - v - for v in self.baseline_simulation.tax_benefit_model_version.variables - if v.name == self.income_variable - ) - - # Get target entity - target_entity = self.entity or var_obj.entity - - # Get data from both simulations - baseline_data = getattr( - self.baseline_simulation.output_dataset.data, target_entity - ) - reform_data = getattr(self.reform_simulation.output_dataset.data, target_entity) - - # Map income variable to target entity if needed - if var_obj.entity != target_entity: - baseline_mapped = ( - self.baseline_simulation.output_dataset.data.map_to_entity( - var_obj.entity, target_entity - ) - ) - baseline_income = baseline_mapped[self.income_variable] - - reform_mapped = self.reform_simulation.output_dataset.data.map_to_entity( - var_obj.entity, target_entity - ) - reform_income = reform_mapped[self.income_variable] - else: - baseline_income = baseline_data[self.income_variable] - reform_income = reform_data[self.income_variable] - - decile_series = calculate_decile_groups( - baseline_data, - baseline_income, + analysis = _prepare_decile_analysis( + self.baseline_simulation, + self.reform_simulation, + income_variable=self.income_variable, decile_variable=self.decile_variable, - entity=target_entity, + entity=self.entity, quantiles=self.quantiles, ) - - # Calculate changes - absolute_change = reform_income - baseline_income - relative_change = (absolute_change / baseline_income) * 100 - - # Filter to this decile - mask = decile_series == self.decile - - # Populate results - self.baseline_mean = float(baseline_income[mask].mean()) - self.reform_mean = float(reform_income[mask].mean()) - self.absolute_change = float(absolute_change[mask].mean()) - self.relative_change = float(relative_change[mask].mean()) - self.count_better_off = float((absolute_change[mask] > 0).sum()) - self.count_worse_off = float((absolute_change[mask] < 0).sum()) - self.count_no_change = float((absolute_change[mask] == 0).sum()) + self._run_from_prepared(analysis) + + def _run_from_prepared( + self, + analysis: _PreparedDecileAnalysis, + ) -> None: + """Populate this output from already validated shared inputs.""" + values = _calculate_decile_impact_values( + analysis, + decile=self.decile, + ) + self.baseline_mean = values.baseline_mean + self.reform_mean = values.reform_mean + self.absolute_change = values.absolute_change + self.relative_change = values.relative_change + self.count_better_off = values.count_better_off + self.count_worse_off = values.count_worse_off + self.count_no_change = values.count_no_change def calculate_decile_impacts( @@ -148,9 +184,19 @@ def calculate_decile_impacts( dynamic=dynamic, ) + assert baseline_simulation is not None + assert reform_simulation is not None baseline_simulation.ensure() reform_simulation.ensure() + analysis = _prepare_decile_analysis( + baseline_simulation, + reform_simulation, + income_variable=income_variable, + decile_variable=decile_variable, + entity=entity, + quantiles=quantiles, + ) results = [] for decile in range(1, quantiles + 1): impact = DecileImpact( @@ -162,7 +208,7 @@ def calculate_decile_impacts( decile=decile, quantiles=quantiles, ) - impact.run() + impact._run_from_prepared(analysis) results.append(impact) # Create DataFrame diff --git a/src/policyengine/outputs/intra_decile_impact.py b/src/policyengine/outputs/intra_decile_impact.py index bfdb6354..165e5be7 100644 --- a/src/policyengine/outputs/intra_decile_impact.py +++ b/src/policyengine/outputs/intra_decile_impact.py @@ -1,7 +1,7 @@ """Intra-decile impact output. Computes the distribution of income change categories within each decile. -Each row represents one decile (1-10) or the overall average (decile=0), +Each row represents one decile (1-10) or the overall result (decile=0), with five proportion columns summing to ~1.0. The five categories classify households by their percentage income change: @@ -15,6 +15,7 @@ household_weight) so they reflect the share of people, not households. """ +from dataclasses import dataclass from typing import Optional import numpy as np @@ -22,7 +23,10 @@ from pydantic import ConfigDict from policyengine.core import Output, OutputCollection, Simulation -from policyengine.outputs.decile_grouping import calculate_decile_groups +from policyengine.outputs.decile_analysis import ( + _prepare_decile_analysis, + _PreparedDecileAnalysis, +) # The 5-category thresholds BOUNDS = [-np.inf, -0.05, -1e-3, 1e-3, 0.05, np.inf] @@ -35,6 +39,58 @@ ] +@dataclass(frozen=True) +class _IntraDecileImpactValues: + """People-weighted category proportions for one selected population.""" + + lose_more_than_5pct: Optional[float] + lose_less_than_5pct: Optional[float] + no_change: Optional[float] + gain_less_than_5pct: Optional[float] + gain_more_than_5pct: Optional[float] + + +def _calculate_intra_decile_values( + analysis: _PreparedDecileAnalysis, + *, + decile: Optional[int], +) -> _IntraDecileImpactValues: + """Calculate category proportions for a decile or the full population.""" + selected = analysis.included.copy() + if decile is not None: + selected &= analysis.groups.eq(decile).fillna(False).to_numpy(dtype=bool) + + selected_weights = analysis.effective_weight[selected] + total_weight = float(np.sum(selected_weights)) + if total_weight == 0: + return _IntraDecileImpactValues( + lose_more_than_5pct=None, + lose_less_than_5pct=None, + no_change=None, + gain_less_than_5pct=None, + gain_more_than_5pct=None, + ) + + baseline_income = analysis.baseline_income[selected] + reform_income = analysis.reform_income[selected] + income_change = (reform_income - baseline_income) / np.maximum( + baseline_income, + 1.0, + ) + proportions = [] + for lower, upper in zip(BOUNDS[:-1], BOUNDS[1:]): + in_category = (income_change > lower) & (income_change <= upper) + proportions.append(float(np.sum(selected_weights[in_category]) / total_weight)) + + return _IntraDecileImpactValues( + lose_more_than_5pct=proportions[0], + lose_less_than_5pct=proportions[1], + no_change=proportions[2], + gain_less_than_5pct=proportions[3], + gain_more_than_5pct=proportions[4], + ) + + class IntraDecileImpact(Output): """Single decile's intra-decile impact — proportion of people in each income change category.""" @@ -56,61 +112,33 @@ class IntraDecileImpact(Output): gain_less_than_5pct: Optional[float] = None gain_more_than_5pct: Optional[float] = None - def run(self): + def run(self) -> None: """Calculate intra-decile proportions for this specific decile.""" - baseline_data = getattr( - self.baseline_simulation.output_dataset.data, self.entity + analysis = _prepare_decile_analysis( + self.baseline_simulation, + self.reform_simulation, + income_variable=self.income_variable, + decile_variable=self.decile_variable, + entity=self.entity, + quantiles=self.quantiles, + require_effective_weight=True, ) - reform_data = getattr(self.reform_simulation.output_dataset.data, self.entity) - - baseline_income_series = baseline_data[self.income_variable] - reform_income_series = reform_data[self.income_variable] - decile_series = np.asarray( - calculate_decile_groups( - baseline_data, - baseline_income_series, - decile_variable=self.decile_variable, - entity=self.entity, - quantiles=self.quantiles, - ) + self._run_from_prepared(analysis) + + def _run_from_prepared( + self, + analysis: _PreparedDecileAnalysis, + ) -> None: + """Populate this output from already validated shared inputs.""" + values = _calculate_intra_decile_values( + analysis, + decile=None if self.decile == 0 else self.decile, ) - baseline_income = np.asarray(baseline_income_series) - reform_income = np.asarray(reform_income_series) - - # People-weighted counts - weights = np.asarray(baseline_data[f"{self.entity}_weight"]) - if self.entity == "household": - people_count = np.asarray(baseline_data["household_count_people"]) - people = people_count * weights - else: - people = weights - - # Compute percentage income change - capped_baseline = np.maximum(baseline_income, 1.0) - income_change = (reform_income - baseline_income) / capped_baseline - - in_decile = decile_series == self.decile - people_in_decile = float(np.sum(people[in_decile])) - - if people_in_decile == 0: - self.lose_more_than_5pct = 0.0 - self.lose_less_than_5pct = 0.0 - self.no_change = 1.0 - self.gain_less_than_5pct = 0.0 - self.gain_more_than_5pct = 0.0 - return - - proportions = [] - for lower, upper in zip(BOUNDS[:-1], BOUNDS[1:]): - in_category = (income_change > lower) & (income_change <= upper) - in_both = in_decile & in_category - proportions.append(float(np.sum(people[in_both]) / people_in_decile)) - - self.lose_more_than_5pct = proportions[0] - self.lose_less_than_5pct = proportions[1] - self.no_change = proportions[2] - self.gain_less_than_5pct = proportions[3] - self.gain_more_than_5pct = proportions[4] + self.lose_more_than_5pct = values.lose_more_than_5pct + self.lose_less_than_5pct = values.lose_less_than_5pct + self.no_change = values.no_change + self.gain_less_than_5pct = values.gain_less_than_5pct + self.gain_more_than_5pct = values.gain_more_than_5pct def compute_intra_decile_impacts( @@ -121,12 +149,21 @@ def compute_intra_decile_impacts( entity: str = "household", quantiles: int = 10, ) -> OutputCollection[IntraDecileImpact]: - """Compute intra-decile proportions for all deciles + overall average. + """Compute intra-decile proportions for all deciles and the population. Returns: OutputCollection containing list of IntraDecileImpact objects - (deciles 1-N plus overall average at decile=0) and DataFrame. + (deciles 1-N plus the direct overall result at decile=0) and DataFrame. """ + analysis = _prepare_decile_analysis( + baseline_simulation, + reform_simulation, + income_variable=income_variable, + decile_variable=decile_variable, + entity=entity, + quantiles=quantiles, + require_effective_weight=True, + ) results = [] for decile in range(1, quantiles + 1): impact = IntraDecileImpact.model_construct( @@ -138,10 +175,9 @@ def compute_intra_decile_impacts( decile=decile, quantiles=quantiles, ) - impact.run() + impact._run_from_prepared(analysis) results.append(impact) - # Overall average (decile=0): arithmetic mean of decile proportions overall = IntraDecileImpact.model_construct( baseline_simulation=baseline_simulation, reform_simulation=reform_simulation, @@ -150,12 +186,8 @@ def compute_intra_decile_impacts( entity=entity, decile=0, quantiles=quantiles, - lose_more_than_5pct=sum(r.lose_more_than_5pct for r in results) / quantiles, - lose_less_than_5pct=sum(r.lose_less_than_5pct for r in results) / quantiles, - no_change=sum(r.no_change for r in results) / quantiles, - gain_less_than_5pct=sum(r.gain_less_than_5pct for r in results) / quantiles, - gain_more_than_5pct=sum(r.gain_more_than_5pct for r in results) / quantiles, ) + overall._run_from_prepared(analysis) results.append(overall) # Create DataFrame