From eec9584b936ddb876d110eed1b1fc2b0c20f6d37 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 12 Apr 2026 17:52:38 -0400 Subject: [PATCH 1/4] Add Scotland and Wales private-rent calibration targets --- changelog.d/316.md | 1 + .../targets/build_loss_matrix.py | 6 +- .../targets/compute/households.py | 40 +++++--- policyengine_uk_data/targets/compute/other.py | 21 +++- .../targets/sources/devolved_housing.py | 96 +++++++++++++++++++ .../tests/test_devolved_housing_targets.py | 75 +++++++++++++++ .../tests/test_target_registry.py | 10 ++ 7 files changed, 227 insertions(+), 22 deletions(-) create mode 100644 changelog.d/316.md create mode 100644 policyengine_uk_data/targets/sources/devolved_housing.py create mode 100644 policyengine_uk_data/tests/test_devolved_housing_targets.py diff --git a/changelog.d/316.md b/changelog.d/316.md new file mode 100644 index 00000000..3addacfb --- /dev/null +++ b/changelog.d/316.md @@ -0,0 +1 @@ +Add Scotland and Wales private-rent country targets so constituency calibration no longer relies only on UK-wide rent anchors for those countries. diff --git a/policyengine_uk_data/targets/build_loss_matrix.py b/policyengine_uk_data/targets/build_loss_matrix.py index e92ecbc1..05def1dc 100644 --- a/policyengine_uk_data/targets/build_loss_matrix.py +++ b/policyengine_uk_data/targets/build_loss_matrix.py @@ -288,10 +288,8 @@ def _compute_column(target: Target, ctx: _SimContext, year: int) -> np.ndarray | return compute_vehicles(target, ctx) # Housing - if name in ( - "housing/total_mortgage", - "housing/rent_private", - "housing/rent_social", + if name == "housing/total_mortgage" or name.startswith( + ("housing/rent_private", "housing/rent_social") ): return compute_housing(target, ctx) diff --git a/policyengine_uk_data/targets/compute/households.py b/policyengine_uk_data/targets/compute/households.py index be51f2d0..0dfe23d4 100644 --- a/policyengine_uk_data/targets/compute/households.py +++ b/policyengine_uk_data/targets/compute/households.py @@ -60,28 +60,38 @@ def ft_hh(value): def compute_tenure(target, ctx) -> np.ndarray | None: """Compute dwelling count by tenure type.""" _TENURE_MAP = { - "tenure_england_owned_outright": "OWNED_OUTRIGHT", - "tenure_england_owned_with_mortgage": "OWNED_WITH_MORTGAGE", - "tenure_england_rented_privately": "RENT_PRIVATELY", - "tenure_england_social_rent": [ - "RENT_FROM_COUNCIL", - "RENT_FROM_HA", - ], - "tenure_england_total": None, + "tenure_england_owned_outright": ("OWNED_OUTRIGHT", "ENGLAND"), + "tenure_england_owned_with_mortgage": ( + "OWNED_WITH_MORTGAGE", + "ENGLAND", + ), + "tenure_england_rented_privately": ("RENT_PRIVATELY", "ENGLAND"), + "tenure_england_social_rent": ( + [ + "RENT_FROM_COUNCIL", + "RENT_FROM_HA", + ], + "ENGLAND", + ), + "tenure_wales_rented_privately": ("RENT_PRIVATELY", "WALES"), + "tenure_scotland_rented_privately": ("RENT_PRIVATELY", "SCOTLAND"), + "tenure_england_total": (None, "ENGLAND"), } - suffix = target.name.removeprefix("ons/") - pe_values = _TENURE_MAP.get(suffix) - if pe_values is None and suffix == "tenure_england_total": - return (ctx.country == "ENGLAND").astype(float) - if pe_values is None: + suffix = target.name.split("/", 1)[-1] + tenure_spec = _TENURE_MAP.get(suffix) + if tenure_spec is None: return None + pe_values, country = tenure_spec + if pe_values is None: + return (ctx.country == country).astype(float) + tenure = ctx.sim.calculate("tenure_type", map_to="household").values - in_england = ctx.country == "ENGLAND" + in_country = ctx.country == country if isinstance(pe_values, list): match = np.zeros_like(tenure, dtype=bool) for v in pe_values: match = match | (tenure == v) else: match = tenure == pe_values - return (match & in_england).astype(float) + return (match & in_country).astype(float) diff --git a/policyengine_uk_data/targets/compute/other.py b/policyengine_uk_data/targets/compute/other.py index 0bd02cb7..6009dd79 100644 --- a/policyengine_uk_data/targets/compute/other.py +++ b/policyengine_uk_data/targets/compute/other.py @@ -15,6 +15,13 @@ "SOUTH_WEST", } +_COUNTRY_CODE_MAP = { + "E": "ENGLAND", + "W": "WALES", + "S": "SCOTLAND", + "N": "NORTHERN_IRELAND", +} + def compute_vehicles(target, ctx) -> np.ndarray: """Compute vehicle ownership targets.""" @@ -33,11 +40,19 @@ def compute_housing(target, ctx) -> np.ndarray: return ctx.pe("mortgage_capital_repayment") + ctx.pe( "mortgage_interest_repayment" ) + + country_mask = 1.0 + if target.geo_code is not None: + country = _COUNTRY_CODE_MAP.get(target.geo_code) + if country is None: + return None + country_mask = (ctx.country == country).astype(float) + tenure = ctx.sim.calculate("tenure_type", map_to="household").values - if name == "housing/rent_social": + if name == "housing/rent_social" or name.startswith("housing/rent_social/"): is_social = (tenure == "RENT_FROM_COUNCIL") | (tenure == "RENT_FROM_HA") - return ctx.pe("rent") * is_social - return ctx.pe("rent") * (tenure == "RENT_PRIVATELY") + return ctx.pe("rent") * is_social * country_mask + return ctx.pe("rent") * (tenure == "RENT_PRIVATELY") * country_mask def compute_savings_interest(target, ctx) -> np.ndarray: diff --git a/policyengine_uk_data/targets/sources/devolved_housing.py b/policyengine_uk_data/targets/sources/devolved_housing.py new file mode 100644 index 00000000..af6a0fbe --- /dev/null +++ b/policyengine_uk_data/targets/sources/devolved_housing.py @@ -0,0 +1,96 @@ +"""Country-level housing targets for Scotland and Wales. + +Adds private-rented stock and private-rent spend anchors for the two +countries that are currently most weakly identified in constituency +calibration. + +Sources: +- Wales dwelling stock by tenure, 31 March 2024: + https://www.gov.wales/dwelling-stock-estimates-31-march-2024-html +- Scotland stock by tenure workbook, 2023: + https://www.gov.scot/binaries/content/documents/govscot/publications/statistics/2018/09/housing-statistics-stock-by-tenure/documents/stock-by-tenure-2017/stock-by-tenure-2017/govscot%3Adocument/Stock%2Bby%2Btenure.xlsx +- ONS private rents bulletin, May 2025: + https://www.ons.gov.uk/economy/inflationandpriceindices/bulletins/privaterentandhousepricesuk/may2025 +""" + +from policyengine_uk_data.targets.schema import GeographicLevel, Target, Unit + +_ONS_RENT_REF = ( + "https://www.ons.gov.uk/economy/inflationandpriceindices/" + "bulletins/privaterentandhousepricesuk/may2025" +) +_WALES_STOCK_REF = ( + "https://www.gov.wales/dwelling-stock-estimates-31-march-2024-html" +) +_SCOTLAND_STOCK_REF = ( + "https://www.gov.scot/binaries/content/documents/govscot/publications/" + "statistics/2018/09/housing-statistics-stock-by-tenure/documents/" + "stock-by-tenure-2017/stock-by-tenure-2017/govscot%3Adocument/" + "Stock%2Bby%2Btenure.xlsx" +) + +_WALES_PRIVATE_RENTED_STOCK_2025 = 200_700 +_SCOTLAND_PRIVATE_RENTED_STOCK_2025 = 357_706 + +_WALES_AVG_MONTHLY_RENT_2025 = 795 +_SCOTLAND_AVG_MONTHLY_RENT_2025 = 999 + +_WALES_PRIVATE_RENT_TOTAL_2025 = ( + _WALES_PRIVATE_RENTED_STOCK_2025 * _WALES_AVG_MONTHLY_RENT_2025 * 12 +) +_SCOTLAND_PRIVATE_RENT_TOTAL_2025 = ( + _SCOTLAND_PRIVATE_RENTED_STOCK_2025 + * _SCOTLAND_AVG_MONTHLY_RENT_2025 + * 12 +) + + +def get_targets() -> list[Target]: + return [ + Target( + name="gov_wales/tenure_wales_rented_privately", + variable="tenure_type", + source="welsh_government", + unit=Unit.COUNT, + geographic_level=GeographicLevel.COUNTRY, + geo_code="W", + geo_name="Wales", + values={2025: _WALES_PRIVATE_RENTED_STOCK_2025}, + is_count=True, + reference_url=_WALES_STOCK_REF, + ), + Target( + name="gov_scot/tenure_scotland_rented_privately", + variable="tenure_type", + source="scottish_government", + unit=Unit.COUNT, + geographic_level=GeographicLevel.COUNTRY, + geo_code="S", + geo_name="Scotland", + values={2025: _SCOTLAND_PRIVATE_RENTED_STOCK_2025}, + is_count=True, + reference_url=_SCOTLAND_STOCK_REF, + ), + Target( + name="housing/rent_private/wales", + variable="rent", + source="ons", + unit=Unit.GBP, + geographic_level=GeographicLevel.COUNTRY, + geo_code="W", + geo_name="Wales", + values={2025: _WALES_PRIVATE_RENT_TOTAL_2025}, + reference_url=_ONS_RENT_REF, + ), + Target( + name="housing/rent_private/scotland", + variable="rent", + source="ons", + unit=Unit.GBP, + geographic_level=GeographicLevel.COUNTRY, + geo_code="S", + geo_name="Scotland", + values={2025: _SCOTLAND_PRIVATE_RENT_TOTAL_2025}, + reference_url=_ONS_RENT_REF, + ), + ] diff --git a/policyengine_uk_data/tests/test_devolved_housing_targets.py b/policyengine_uk_data/tests/test_devolved_housing_targets.py new file mode 100644 index 00000000..959497b4 --- /dev/null +++ b/policyengine_uk_data/tests/test_devolved_housing_targets.py @@ -0,0 +1,75 @@ +from types import SimpleNamespace + +import numpy as np + +from policyengine_uk_data.targets.compute.households import compute_tenure +from policyengine_uk_data.targets.compute.other import compute_housing +from policyengine_uk_data.targets.sources.devolved_housing import ( + _SCOTLAND_PRIVATE_RENT_TOTAL_2025, + _WALES_PRIVATE_RENT_TOTAL_2025, + get_targets, +) + + +class _FakeSim: + def __init__(self, tenure_type): + self._tenure_type = np.array(tenure_type) + + def calculate(self, variable, map_to=None): + if variable != "tenure_type" or map_to != "household": + raise AssertionError(f"Unexpected calculate call: {variable}, {map_to}") + return SimpleNamespace(values=self._tenure_type) + + +class _FakeCtx: + def __init__(self, tenure_type, country, rent): + self.sim = _FakeSim(tenure_type) + self.country = np.array(country) + self._rent = np.array(rent) + + def pe(self, variable): + if variable != "rent": + raise AssertionError(f"Unexpected pe call: {variable}") + return self._rent + + +def test_devolved_targets_exist(): + names = {t.name for t in get_targets()} + assert "gov_scot/tenure_scotland_rented_privately" in names + assert "gov_wales/tenure_wales_rented_privately" in names + assert "housing/rent_private/scotland" in names + assert "housing/rent_private/wales" in names + + +def test_devolved_private_rent_values(): + targets = {t.name: t for t in get_targets()} + assert ( + targets["housing/rent_private/scotland"].values[2025] + == _SCOTLAND_PRIVATE_RENT_TOTAL_2025 + ) + assert ( + targets["housing/rent_private/wales"].values[2025] + == _WALES_PRIVATE_RENT_TOTAL_2025 + ) + + +def test_compute_housing_filters_to_country(): + ctx = _FakeCtx( + tenure_type=["RENT_PRIVATELY", "RENT_PRIVATELY", "RENT_FROM_HA"], + country=["SCOTLAND", "WALES", "SCOTLAND"], + rent=[1000.0, 800.0, 600.0], + ) + target = SimpleNamespace(name="housing/rent_private/scotland", geo_code="S") + result = compute_housing(target, ctx) + np.testing.assert_array_equal(result, np.array([1000.0, 0.0, 0.0])) + + +def test_compute_tenure_filters_to_country(): + ctx = _FakeCtx( + tenure_type=["RENT_PRIVATELY", "RENT_PRIVATELY", "OWNED_OUTRIGHT"], + country=["SCOTLAND", "WALES", "SCOTLAND"], + rent=[0.0, 0.0, 0.0], + ) + target = SimpleNamespace(name="gov_scot/tenure_scotland_rented_privately") + result = compute_tenure(target, ctx) + np.testing.assert_array_equal(result, np.array([1.0, 0.0, 0.0])) diff --git a/policyengine_uk_data/tests/test_target_registry.py b/policyengine_uk_data/tests/test_target_registry.py index 51e9cc69..3f2b2305 100644 --- a/policyengine_uk_data/tests/test_target_registry.py +++ b/policyengine_uk_data/tests/test_target_registry.py @@ -129,3 +129,13 @@ def test_land_targets_exist(): assert "ons/household_land_value" in names assert "ons/corporate_land_value" in names assert "ons/land_value" in names + + +def test_devolved_private_rent_targets_exist(): + """Country-level private-rent anchors should exist for Scotland and Wales.""" + targets = get_all_targets(year=2025) + names = {t.name for t in targets} + assert "gov_scot/tenure_scotland_rented_privately" in names + assert "gov_wales/tenure_wales_rented_privately" in names + assert "housing/rent_private/scotland" in names + assert "housing/rent_private/wales" in names From 85334d70d819f73505cff9c1a8910059f7f666d7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 12 Apr 2026 17:53:43 -0400 Subject: [PATCH 2/4] Format devolved housing targets --- policyengine_uk_data/targets/sources/devolved_housing.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/policyengine_uk_data/targets/sources/devolved_housing.py b/policyengine_uk_data/targets/sources/devolved_housing.py index af6a0fbe..17ad2c2c 100644 --- a/policyengine_uk_data/targets/sources/devolved_housing.py +++ b/policyengine_uk_data/targets/sources/devolved_housing.py @@ -19,9 +19,7 @@ "https://www.ons.gov.uk/economy/inflationandpriceindices/" "bulletins/privaterentandhousepricesuk/may2025" ) -_WALES_STOCK_REF = ( - "https://www.gov.wales/dwelling-stock-estimates-31-march-2024-html" -) +_WALES_STOCK_REF = "https://www.gov.wales/dwelling-stock-estimates-31-march-2024-html" _SCOTLAND_STOCK_REF = ( "https://www.gov.scot/binaries/content/documents/govscot/publications/" "statistics/2018/09/housing-statistics-stock-by-tenure/documents/" @@ -39,9 +37,7 @@ _WALES_PRIVATE_RENTED_STOCK_2025 * _WALES_AVG_MONTHLY_RENT_2025 * 12 ) _SCOTLAND_PRIVATE_RENT_TOTAL_2025 = ( - _SCOTLAND_PRIVATE_RENTED_STOCK_2025 - * _SCOTLAND_AVG_MONTHLY_RENT_2025 - * 12 + _SCOTLAND_PRIVATE_RENTED_STOCK_2025 * _SCOTLAND_AVG_MONTHLY_RENT_2025 * 12 ) From 571d134b1ec1a63a56b6f11e5ae951a31aad9a4f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 12 Apr 2026 17:59:14 -0400 Subject: [PATCH 3/4] Use rent-average constraints for devolved housing targets --- .../targets/build_loss_matrix.py | 6 ++- policyengine_uk_data/targets/compute/other.py | 20 ++------- .../targets/sources/devolved_housing.py | 41 +++++++++++++------ .../tests/test_devolved_housing_targets.py | 32 ++++++--------- .../tests/test_target_registry.py | 4 +- 5 files changed, 50 insertions(+), 53 deletions(-) diff --git a/policyengine_uk_data/targets/build_loss_matrix.py b/policyengine_uk_data/targets/build_loss_matrix.py index 05def1dc..e92ecbc1 100644 --- a/policyengine_uk_data/targets/build_loss_matrix.py +++ b/policyengine_uk_data/targets/build_loss_matrix.py @@ -288,8 +288,10 @@ def _compute_column(target: Target, ctx: _SimContext, year: int) -> np.ndarray | return compute_vehicles(target, ctx) # Housing - if name == "housing/total_mortgage" or name.startswith( - ("housing/rent_private", "housing/rent_social") + if name in ( + "housing/total_mortgage", + "housing/rent_private", + "housing/rent_social", ): return compute_housing(target, ctx) diff --git a/policyengine_uk_data/targets/compute/other.py b/policyengine_uk_data/targets/compute/other.py index 6009dd79..25c01eb1 100644 --- a/policyengine_uk_data/targets/compute/other.py +++ b/policyengine_uk_data/targets/compute/other.py @@ -15,13 +15,6 @@ "SOUTH_WEST", } -_COUNTRY_CODE_MAP = { - "E": "ENGLAND", - "W": "WALES", - "S": "SCOTLAND", - "N": "NORTHERN_IRELAND", -} - def compute_vehicles(target, ctx) -> np.ndarray: """Compute vehicle ownership targets.""" @@ -41,18 +34,11 @@ def compute_housing(target, ctx) -> np.ndarray: "mortgage_interest_repayment" ) - country_mask = 1.0 - if target.geo_code is not None: - country = _COUNTRY_CODE_MAP.get(target.geo_code) - if country is None: - return None - country_mask = (ctx.country == country).astype(float) - tenure = ctx.sim.calculate("tenure_type", map_to="household").values - if name == "housing/rent_social" or name.startswith("housing/rent_social/"): + if name == "housing/rent_social": is_social = (tenure == "RENT_FROM_COUNCIL") | (tenure == "RENT_FROM_HA") - return ctx.pe("rent") * is_social * country_mask - return ctx.pe("rent") * (tenure == "RENT_PRIVATELY") * country_mask + return ctx.pe("rent") * is_social + return ctx.pe("rent") * (tenure == "RENT_PRIVATELY") def compute_savings_interest(target, ctx) -> np.ndarray: diff --git a/policyengine_uk_data/targets/sources/devolved_housing.py b/policyengine_uk_data/targets/sources/devolved_housing.py index 17ad2c2c..3f7459b8 100644 --- a/policyengine_uk_data/targets/sources/devolved_housing.py +++ b/policyengine_uk_data/targets/sources/devolved_housing.py @@ -1,8 +1,7 @@ """Country-level housing targets for Scotland and Wales. -Adds private-rented stock and private-rent spend anchors for the two -countries that are currently most weakly identified in constituency -calibration. +Adds private-rented stock and average-rent anchors for the two countries +that are currently most weakly identified in constituency calibration. Sources: - Wales dwelling stock by tenure, 31 March 2024: @@ -13,6 +12,8 @@ https://www.ons.gov.uk/economy/inflationandpriceindices/bulletins/privaterentandhousepricesuk/may2025 """ +import numpy as np + from policyengine_uk_data.targets.schema import GeographicLevel, Target, Unit _ONS_RENT_REF = ( @@ -33,12 +34,24 @@ _WALES_AVG_MONTHLY_RENT_2025 = 795 _SCOTLAND_AVG_MONTHLY_RENT_2025 = 999 -_WALES_PRIVATE_RENT_TOTAL_2025 = ( - _WALES_PRIVATE_RENTED_STOCK_2025 * _WALES_AVG_MONTHLY_RENT_2025 * 12 -) -_SCOTLAND_PRIVATE_RENT_TOTAL_2025 = ( - _SCOTLAND_PRIVATE_RENTED_STOCK_2025 * _SCOTLAND_AVG_MONTHLY_RENT_2025 * 12 -) +_COUNTRY_NAME = { + "W": "WALES", + "S": "SCOTLAND", +} +_COUNTRY_ANNUAL_PRIVATE_RENT = { + "W": _WALES_AVG_MONTHLY_RENT_2025 * 12, + "S": _SCOTLAND_AVG_MONTHLY_RENT_2025 * 12, +} + + +def _compute_private_rent_average_gap(ctx, target, year) -> np.ndarray: + """Linearised country private-rent average constraint.""" + country = _COUNTRY_NAME[target.geo_code] + annual_rent = _COUNTRY_ANNUAL_PRIVATE_RENT[target.geo_code] + tenure = ctx.sim.calculate("tenure_type", map_to="household").values + private_renter = tenure == "RENT_PRIVATELY" + in_country = ctx.country == country + return np.where(in_country & private_renter, ctx.pe("rent") - annual_rent, 0.0) def get_targets() -> list[Target]: @@ -68,25 +81,27 @@ def get_targets() -> list[Target]: reference_url=_SCOTLAND_STOCK_REF, ), Target( - name="housing/rent_private/wales", + name="housing/private_rent_average_gap/wales", variable="rent", source="ons", unit=Unit.GBP, geographic_level=GeographicLevel.COUNTRY, geo_code="W", geo_name="Wales", - values={2025: _WALES_PRIVATE_RENT_TOTAL_2025}, + values={2025: 0.0}, reference_url=_ONS_RENT_REF, + custom_compute=_compute_private_rent_average_gap, ), Target( - name="housing/rent_private/scotland", + name="housing/private_rent_average_gap/scotland", variable="rent", source="ons", unit=Unit.GBP, geographic_level=GeographicLevel.COUNTRY, geo_code="S", geo_name="Scotland", - values={2025: _SCOTLAND_PRIVATE_RENT_TOTAL_2025}, + values={2025: 0.0}, reference_url=_ONS_RENT_REF, + custom_compute=_compute_private_rent_average_gap, ), ] diff --git a/policyengine_uk_data/tests/test_devolved_housing_targets.py b/policyengine_uk_data/tests/test_devolved_housing_targets.py index 959497b4..30e8fcfd 100644 --- a/policyengine_uk_data/tests/test_devolved_housing_targets.py +++ b/policyengine_uk_data/tests/test_devolved_housing_targets.py @@ -3,10 +3,9 @@ import numpy as np from policyengine_uk_data.targets.compute.households import compute_tenure -from policyengine_uk_data.targets.compute.other import compute_housing from policyengine_uk_data.targets.sources.devolved_housing import ( - _SCOTLAND_PRIVATE_RENT_TOTAL_2025, - _WALES_PRIVATE_RENT_TOTAL_2025, + _COUNTRY_ANNUAL_PRIVATE_RENT, + _compute_private_rent_average_gap, get_targets, ) @@ -37,31 +36,26 @@ def test_devolved_targets_exist(): names = {t.name for t in get_targets()} assert "gov_scot/tenure_scotland_rented_privately" in names assert "gov_wales/tenure_wales_rented_privately" in names - assert "housing/rent_private/scotland" in names - assert "housing/rent_private/wales" in names + assert "housing/private_rent_average_gap/scotland" in names + assert "housing/private_rent_average_gap/wales" in names -def test_devolved_private_rent_values(): +def test_devolved_private_rent_gap_targets_zero(): targets = {t.name: t for t in get_targets()} - assert ( - targets["housing/rent_private/scotland"].values[2025] - == _SCOTLAND_PRIVATE_RENT_TOTAL_2025 - ) - assert ( - targets["housing/rent_private/wales"].values[2025] - == _WALES_PRIVATE_RENT_TOTAL_2025 - ) + assert targets["housing/private_rent_average_gap/scotland"].values[2025] == 0 + assert targets["housing/private_rent_average_gap/wales"].values[2025] == 0 -def test_compute_housing_filters_to_country(): +def test_compute_private_rent_average_gap_filters_to_country(): ctx = _FakeCtx( tenure_type=["RENT_PRIVATELY", "RENT_PRIVATELY", "RENT_FROM_HA"], country=["SCOTLAND", "WALES", "SCOTLAND"], - rent=[1000.0, 800.0, 600.0], + rent=[12_600.0, 9_600.0, 600.0], ) - target = SimpleNamespace(name="housing/rent_private/scotland", geo_code="S") - result = compute_housing(target, ctx) - np.testing.assert_array_equal(result, np.array([1000.0, 0.0, 0.0])) + target = SimpleNamespace(geo_code="S") + result = _compute_private_rent_average_gap(ctx, target, 2025) + expected_gap = 12_600.0 - _COUNTRY_ANNUAL_PRIVATE_RENT["S"] + np.testing.assert_array_equal(result, np.array([expected_gap, 0.0, 0.0])) def test_compute_tenure_filters_to_country(): diff --git a/policyengine_uk_data/tests/test_target_registry.py b/policyengine_uk_data/tests/test_target_registry.py index 3f2b2305..973c23c9 100644 --- a/policyengine_uk_data/tests/test_target_registry.py +++ b/policyengine_uk_data/tests/test_target_registry.py @@ -137,5 +137,5 @@ def test_devolved_private_rent_targets_exist(): names = {t.name for t in targets} assert "gov_scot/tenure_scotland_rented_privately" in names assert "gov_wales/tenure_wales_rented_privately" in names - assert "housing/rent_private/scotland" in names - assert "housing/rent_private/wales" in names + assert "housing/private_rent_average_gap/scotland" in names + assert "housing/private_rent_average_gap/wales" in names From 434b951eedd55c5038a13bca1c20ea097964e21f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 13 Apr 2026 06:47:51 -0400 Subject: [PATCH 4/4] Wire devolved rent anchors into constituency calibration --- .../constituencies/devolved_housing.py | 76 ++++++ .../local_areas/constituencies/loss.py | 13 + .../targets/compute/households.py | 2 - .../targets/sources/devolved_housing.py | 107 --------- .../tests/test_devolved_housing_targets.py | 225 ++++++++++++++---- .../tests/test_target_registry.py | 10 - 6 files changed, 264 insertions(+), 169 deletions(-) create mode 100644 policyengine_uk_data/datasets/local_areas/constituencies/devolved_housing.py delete mode 100644 policyengine_uk_data/targets/sources/devolved_housing.py diff --git a/policyengine_uk_data/datasets/local_areas/constituencies/devolved_housing.py b/policyengine_uk_data/datasets/local_areas/constituencies/devolved_housing.py new file mode 100644 index 00000000..0b03e87a --- /dev/null +++ b/policyengine_uk_data/datasets/local_areas/constituencies/devolved_housing.py @@ -0,0 +1,76 @@ +"""Country-level private-rent anchors for constituency calibration. + +These are used directly by the constituency loss builder rather than the +general target registry, because constituency calibration uses a bespoke +matrix constructor. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + + +_PRIVATE_RENT_TARGETS = { + "WALES": { + "private_renter_households": 200_700, + "annual_private_rent": 795 * 12, + }, + "SCOTLAND": { + "private_renter_households": 357_706, + "annual_private_rent": 999 * 12, + }, +} + +_CODE_TO_COUNTRY = { + "W": "WALES", + "S": "SCOTLAND", +} + + +def add_private_rent_targets( + matrix: pd.DataFrame, + y: pd.DataFrame, + age_targets: pd.DataFrame, + *, + country: np.ndarray, + tenure_type: np.ndarray, + rent: np.ndarray, +) -> None: + """Append Wales/Scotland private-rent count and amount targets. + + Country totals are allocated across 2010 constituencies in proportion to + their official age-target population shares within each country. + """ + + constituency_population = age_targets.filter(like="age/").sum(axis=1) + constituency_country = age_targets["code"].str[0].map(_CODE_TO_COUNTRY) + private_renter = tenure_type == "RENT_PRIVATELY" + + for country_name, target in _PRIVATE_RENT_TARGETS.items(): + area_mask = constituency_country == country_name + country_population = constituency_population.where(area_mask, 0).sum() + if country_population <= 0: + raise ValueError( + f"No constituency population available for {country_name} housing targets" + ) + + share = np.where(area_mask, constituency_population / country_population, 0.0) + in_country_private_rent = (country == country_name) & private_renter + prefix = country_name.lower() + + matrix[f"housing/{prefix}_private_renter_households"] = ( + in_country_private_rent + ).astype(float) + matrix[f"housing/{prefix}_private_rent_amount"] = np.where( + in_country_private_rent, + rent, + 0.0, + ) + + y[f"housing/{prefix}_private_renter_households"] = ( + share * target["private_renter_households"] + ) + y[f"housing/{prefix}_private_rent_amount"] = ( + share * target["private_renter_households"] * target["annual_private_rent"] + ) diff --git a/policyengine_uk_data/datasets/local_areas/constituencies/loss.py b/policyengine_uk_data/datasets/local_areas/constituencies/loss.py index 3ea6e12a..c663a34b 100644 --- a/policyengine_uk_data/datasets/local_areas/constituencies/loss.py +++ b/policyengine_uk_data/datasets/local_areas/constituencies/loss.py @@ -32,6 +32,9 @@ get_constituency_uc_targets, get_constituency_uc_by_children_targets, ) +from policyengine_uk_data.datasets.local_areas.constituencies.devolved_housing import ( + add_private_rent_targets, +) def create_constituency_target_matrix( @@ -114,6 +117,16 @@ def create_constituency_target_matrix( for col in uc_by_children.columns: y[col] = uc_by_children[col].values + # ── Wales/Scotland housing anchors ─────────────────────────────── + add_private_rent_targets( + matrix, + y, + age_targets, + country=sim.calculate("country").values, + tenure_type=sim.calculate("tenure_type").values, + rent=sim.calculate("rent").values, + ) + # ── Boundary mapping (2010 → 2024) ──────────────────────────── const_2024 = pd.read_csv(STORAGE_FOLDER / "constituencies_2024.csv") diff --git a/policyengine_uk_data/targets/compute/households.py b/policyengine_uk_data/targets/compute/households.py index 0dfe23d4..68719fc1 100644 --- a/policyengine_uk_data/targets/compute/households.py +++ b/policyengine_uk_data/targets/compute/households.py @@ -73,8 +73,6 @@ def compute_tenure(target, ctx) -> np.ndarray | None: ], "ENGLAND", ), - "tenure_wales_rented_privately": ("RENT_PRIVATELY", "WALES"), - "tenure_scotland_rented_privately": ("RENT_PRIVATELY", "SCOTLAND"), "tenure_england_total": (None, "ENGLAND"), } suffix = target.name.split("/", 1)[-1] diff --git a/policyengine_uk_data/targets/sources/devolved_housing.py b/policyengine_uk_data/targets/sources/devolved_housing.py deleted file mode 100644 index 3f7459b8..00000000 --- a/policyengine_uk_data/targets/sources/devolved_housing.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Country-level housing targets for Scotland and Wales. - -Adds private-rented stock and average-rent anchors for the two countries -that are currently most weakly identified in constituency calibration. - -Sources: -- Wales dwelling stock by tenure, 31 March 2024: - https://www.gov.wales/dwelling-stock-estimates-31-march-2024-html -- Scotland stock by tenure workbook, 2023: - https://www.gov.scot/binaries/content/documents/govscot/publications/statistics/2018/09/housing-statistics-stock-by-tenure/documents/stock-by-tenure-2017/stock-by-tenure-2017/govscot%3Adocument/Stock%2Bby%2Btenure.xlsx -- ONS private rents bulletin, May 2025: - https://www.ons.gov.uk/economy/inflationandpriceindices/bulletins/privaterentandhousepricesuk/may2025 -""" - -import numpy as np - -from policyengine_uk_data.targets.schema import GeographicLevel, Target, Unit - -_ONS_RENT_REF = ( - "https://www.ons.gov.uk/economy/inflationandpriceindices/" - "bulletins/privaterentandhousepricesuk/may2025" -) -_WALES_STOCK_REF = "https://www.gov.wales/dwelling-stock-estimates-31-march-2024-html" -_SCOTLAND_STOCK_REF = ( - "https://www.gov.scot/binaries/content/documents/govscot/publications/" - "statistics/2018/09/housing-statistics-stock-by-tenure/documents/" - "stock-by-tenure-2017/stock-by-tenure-2017/govscot%3Adocument/" - "Stock%2Bby%2Btenure.xlsx" -) - -_WALES_PRIVATE_RENTED_STOCK_2025 = 200_700 -_SCOTLAND_PRIVATE_RENTED_STOCK_2025 = 357_706 - -_WALES_AVG_MONTHLY_RENT_2025 = 795 -_SCOTLAND_AVG_MONTHLY_RENT_2025 = 999 - -_COUNTRY_NAME = { - "W": "WALES", - "S": "SCOTLAND", -} -_COUNTRY_ANNUAL_PRIVATE_RENT = { - "W": _WALES_AVG_MONTHLY_RENT_2025 * 12, - "S": _SCOTLAND_AVG_MONTHLY_RENT_2025 * 12, -} - - -def _compute_private_rent_average_gap(ctx, target, year) -> np.ndarray: - """Linearised country private-rent average constraint.""" - country = _COUNTRY_NAME[target.geo_code] - annual_rent = _COUNTRY_ANNUAL_PRIVATE_RENT[target.geo_code] - tenure = ctx.sim.calculate("tenure_type", map_to="household").values - private_renter = tenure == "RENT_PRIVATELY" - in_country = ctx.country == country - return np.where(in_country & private_renter, ctx.pe("rent") - annual_rent, 0.0) - - -def get_targets() -> list[Target]: - return [ - Target( - name="gov_wales/tenure_wales_rented_privately", - variable="tenure_type", - source="welsh_government", - unit=Unit.COUNT, - geographic_level=GeographicLevel.COUNTRY, - geo_code="W", - geo_name="Wales", - values={2025: _WALES_PRIVATE_RENTED_STOCK_2025}, - is_count=True, - reference_url=_WALES_STOCK_REF, - ), - Target( - name="gov_scot/tenure_scotland_rented_privately", - variable="tenure_type", - source="scottish_government", - unit=Unit.COUNT, - geographic_level=GeographicLevel.COUNTRY, - geo_code="S", - geo_name="Scotland", - values={2025: _SCOTLAND_PRIVATE_RENTED_STOCK_2025}, - is_count=True, - reference_url=_SCOTLAND_STOCK_REF, - ), - Target( - name="housing/private_rent_average_gap/wales", - variable="rent", - source="ons", - unit=Unit.GBP, - geographic_level=GeographicLevel.COUNTRY, - geo_code="W", - geo_name="Wales", - values={2025: 0.0}, - reference_url=_ONS_RENT_REF, - custom_compute=_compute_private_rent_average_gap, - ), - Target( - name="housing/private_rent_average_gap/scotland", - variable="rent", - source="ons", - unit=Unit.GBP, - geographic_level=GeographicLevel.COUNTRY, - geo_code="S", - geo_name="Scotland", - values={2025: 0.0}, - reference_url=_ONS_RENT_REF, - custom_compute=_compute_private_rent_average_gap, - ), - ] diff --git a/policyengine_uk_data/tests/test_devolved_housing_targets.py b/policyengine_uk_data/tests/test_devolved_housing_targets.py index 30e8fcfd..0619bd93 100644 --- a/policyengine_uk_data/tests/test_devolved_housing_targets.py +++ b/policyengine_uk_data/tests/test_devolved_housing_targets.py @@ -1,69 +1,194 @@ -from types import SimpleNamespace - import numpy as np +import pandas as pd -from policyengine_uk_data.targets.compute.households import compute_tenure -from policyengine_uk_data.targets.sources.devolved_housing import ( - _COUNTRY_ANNUAL_PRIVATE_RENT, - _compute_private_rent_average_gap, - get_targets, +from policyengine_uk_data.datasets.local_areas.constituencies.devolved_housing import ( + _PRIVATE_RENT_TARGETS, + add_private_rent_targets, +) +from policyengine_uk_data.datasets.local_areas.constituencies import ( + loss as constituency_loss, ) -class _FakeSim: - def __init__(self, tenure_type): - self._tenure_type = np.array(tenure_type) +def _age_targets(): + return pd.DataFrame( + { + "code": ["W07000041", "W07000042", "S14000001", "S14000002"], + "name": ["W1", "W2", "S1", "S2"], + "age/0_10": [100, 300, 200, 200], + "age/10_20": [100, 300, 200, 200], + } + ) - def calculate(self, variable, map_to=None): - if variable != "tenure_type" or map_to != "household": - raise AssertionError(f"Unexpected calculate call: {variable}, {map_to}") - return SimpleNamespace(values=self._tenure_type) +def test_add_private_rent_targets_filters_matrix_to_country_private_renters(): + matrix = pd.DataFrame() + y = pd.DataFrame() -class _FakeCtx: - def __init__(self, tenure_type, country, rent): - self.sim = _FakeSim(tenure_type) - self.country = np.array(country) - self._rent = np.array(rent) + add_private_rent_targets( + matrix, + y, + _age_targets(), + country=np.array(["WALES", "WALES", "SCOTLAND", "ENGLAND"]), + tenure_type=np.array( + ["RENT_PRIVATELY", "OWNED_OUTRIGHT", "RENT_PRIVATELY", "RENT_PRIVATELY"] + ), + rent=np.array([9_600.0, 0.0, 12_000.0, 15_000.0]), + ) + + np.testing.assert_array_equal( + matrix["housing/wales_private_renter_households"].values, + np.array([1.0, 0.0, 0.0, 0.0]), + ) + np.testing.assert_array_equal( + matrix["housing/scotland_private_renter_households"].values, + np.array([0.0, 0.0, 1.0, 0.0]), + ) + np.testing.assert_array_equal( + matrix["housing/wales_private_rent_amount"].values, + np.array([9_600.0, 0.0, 0.0, 0.0]), + ) + np.testing.assert_array_equal( + matrix["housing/scotland_private_rent_amount"].values, + np.array([0.0, 0.0, 12_000.0, 0.0]), + ) - def pe(self, variable): - if variable != "rent": - raise AssertionError(f"Unexpected pe call: {variable}") - return self._rent +def test_add_private_rent_targets_allocate_country_totals_by_population_share(): + matrix = pd.DataFrame() + y = pd.DataFrame() + + add_private_rent_targets( + matrix, + y, + _age_targets(), + country=np.array(["WALES", "SCOTLAND"]), + tenure_type=np.array(["RENT_PRIVATELY", "RENT_PRIVATELY"]), + rent=np.array([9_600.0, 12_000.0]), + ) + + wales_shares = np.array([0.25, 0.75, 0.0, 0.0]) + scotland_shares = np.array([0.0, 0.0, 0.5, 0.5]) + + np.testing.assert_allclose( + y["housing/wales_private_renter_households"].values, + wales_shares * _PRIVATE_RENT_TARGETS["WALES"]["private_renter_households"], + ) + np.testing.assert_allclose( + y["housing/scotland_private_renter_households"].values, + scotland_shares + * _PRIVATE_RENT_TARGETS["SCOTLAND"]["private_renter_households"], + ) + np.testing.assert_allclose( + y["housing/wales_private_rent_amount"].values.sum(), + _PRIVATE_RENT_TARGETS["WALES"]["private_renter_households"] + * _PRIVATE_RENT_TARGETS["WALES"]["annual_private_rent"], + ) + np.testing.assert_allclose( + y["housing/scotland_private_rent_amount"].values.sum(), + _PRIVATE_RENT_TARGETS["SCOTLAND"]["private_renter_households"] + * _PRIVATE_RENT_TARGETS["SCOTLAND"]["annual_private_rent"], + ) -def test_devolved_targets_exist(): - names = {t.name for t in get_targets()} - assert "gov_scot/tenure_scotland_rented_privately" in names - assert "gov_wales/tenure_wales_rented_privately" in names - assert "housing/private_rent_average_gap/scotland" in names - assert "housing/private_rent_average_gap/wales" in names +class _FakeDataset: + time_period = 2025 -def test_devolved_private_rent_gap_targets_zero(): - targets = {t.name: t for t in get_targets()} - assert targets["housing/private_rent_average_gap/scotland"].values[2025] == 0 - assert targets["housing/private_rent_average_gap/wales"].values[2025] == 0 +class _FakeSim: + def __init__(self, *args, **kwargs): + self.default_calculation_period = 2025 + + def calculate(self, variable): + mapping = { + "self_employment_income": np.array([0.0, 0.0]), + "employment_income": np.array([0.0, 0.0]), + "income_tax": np.array([1.0, 1.0]), + "age": np.array([35, 35]), + "universal_credit": np.array([1.0, 1.0]), + "is_child": np.array([0.0, 0.0]), + "country": np.array(["WALES", "SCOTLAND"]), + "tenure_type": np.array(["RENT_PRIVATELY", "RENT_PRIVATELY"]), + "rent": np.array([9_600.0, 12_000.0]), + } + return type("Result", (), {"values": mapping[variable]})() + + def map_result(self, values, source_entity, target_entity): + return np.asarray(values) + + +def test_constituency_target_matrix_includes_devolved_housing_targets(monkeypatch): + age_targets = _age_targets().iloc[[0, 2]].reset_index(drop=True) + income_targets = pd.DataFrame( + { + "self_employment_income_amount": [1.0, 1.0], + "self_employment_income_count": [1.0, 1.0], + "employment_income_amount": [1.0, 1.0], + "employment_income_count": [1.0, 1.0], + } + ) + national_income = pd.DataFrame( + { + "total_income_lower_bound": [12_570], + "total_income_upper_bound": [np.inf], + "self_employment_income_amount": [1.0], + "employment_income_amount": [1.0], + } + ) + uc_by_children = pd.DataFrame( + { + "uc_hh_0_children": [1.0, 1.0], + "uc_hh_1_child": [0.0, 0.0], + "uc_hh_2_children": [0.0, 0.0], + "uc_hh_3plus_children": [0.0, 0.0], + } + ) -def test_compute_private_rent_average_gap_filters_to_country(): - ctx = _FakeCtx( - tenure_type=["RENT_PRIVATELY", "RENT_PRIVATELY", "RENT_FROM_HA"], - country=["SCOTLAND", "WALES", "SCOTLAND"], - rent=[12_600.0, 9_600.0, 600.0], + monkeypatch.setattr(constituency_loss, "Microsimulation", _FakeSim) + monkeypatch.setattr( + constituency_loss, "get_constituency_income_targets", lambda: income_targets + ) + monkeypatch.setattr( + constituency_loss, + "get_national_income_projections", + lambda year: national_income, + ) + monkeypatch.setattr( + constituency_loss, "get_constituency_age_targets", lambda: age_targets + ) + monkeypatch.setattr(constituency_loss, "get_uk_total_population", lambda year: 2.0) + monkeypatch.setattr( + constituency_loss, + "get_constituency_uc_targets", + lambda: pd.Series([1.0, 1.0]), + ) + monkeypatch.setattr( + constituency_loss, + "get_constituency_uc_by_children_targets", + lambda: uc_by_children, + ) + monkeypatch.setattr(constituency_loss, "mapping_matrix", np.eye(2)) + monkeypatch.setattr( + constituency_loss.pd, + "read_csv", + lambda path: pd.DataFrame({"code": ["W07000041", "S14000001"]}), ) - target = SimpleNamespace(geo_code="S") - result = _compute_private_rent_average_gap(ctx, target, 2025) - expected_gap = 12_600.0 - _COUNTRY_ANNUAL_PRIVATE_RENT["S"] - np.testing.assert_array_equal(result, np.array([expected_gap, 0.0, 0.0])) + matrix, y, country_mask = constituency_loss.create_constituency_target_matrix( + _FakeDataset() + ) -def test_compute_tenure_filters_to_country(): - ctx = _FakeCtx( - tenure_type=["RENT_PRIVATELY", "RENT_PRIVATELY", "OWNED_OUTRIGHT"], - country=["SCOTLAND", "WALES", "SCOTLAND"], - rent=[0.0, 0.0, 0.0], + assert "housing/wales_private_renter_households" in matrix.columns + assert "housing/scotland_private_rent_amount" in matrix.columns + np.testing.assert_allclose( + y["housing/wales_private_renter_households"].values, + np.array([_PRIVATE_RENT_TARGETS["WALES"]["private_renter_households"], 0.0]), + ) + np.testing.assert_allclose( + y["housing/scotland_private_renter_households"].values, + np.array([0.0, _PRIVATE_RENT_TARGETS["SCOTLAND"]["private_renter_households"]]), + ) + np.testing.assert_array_equal( + country_mask, + np.array([[1.0, 0.0], [0.0, 1.0]]), ) - target = SimpleNamespace(name="gov_scot/tenure_scotland_rented_privately") - result = compute_tenure(target, ctx) - np.testing.assert_array_equal(result, np.array([1.0, 0.0, 0.0])) diff --git a/policyengine_uk_data/tests/test_target_registry.py b/policyengine_uk_data/tests/test_target_registry.py index 973c23c9..51e9cc69 100644 --- a/policyengine_uk_data/tests/test_target_registry.py +++ b/policyengine_uk_data/tests/test_target_registry.py @@ -129,13 +129,3 @@ def test_land_targets_exist(): assert "ons/household_land_value" in names assert "ons/corporate_land_value" in names assert "ons/land_value" in names - - -def test_devolved_private_rent_targets_exist(): - """Country-level private-rent anchors should exist for Scotland and Wales.""" - targets = get_all_targets(year=2025) - names = {t.name for t in targets} - assert "gov_scot/tenure_scotland_rented_privately" in names - assert "gov_wales/tenure_wales_rented_privately" in names - assert "housing/private_rent_average_gap/scotland" in names - assert "housing/private_rent_average_gap/wales" in names