Add Scotland and Wales private-rent calibration targets

changelog.d/316.md

@@ -0,0 +1 @@
+Add Scotland and Wales private-rent country targets so constituency calibration no longer relies only on UK-wide rent anchors for those countries.

policyengine_uk_data/datasets/local_areas/constituencies/devolved_housing.py

@@ -0,0 +1,76 @@
+"""Country-level private-rent anchors for constituency calibration.
+
+These are used directly by the constituency loss builder rather than the
+general target registry, because constituency calibration uses a bespoke
+matrix constructor.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+
+_PRIVATE_RENT_TARGETS = {
+    "WALES": {
+        "private_renter_households": 200_700,
+        "annual_private_rent": 795 * 12,
+    },
+    "SCOTLAND": {
+        "private_renter_households": 357_706,
+        "annual_private_rent": 999 * 12,
+    },
+}
+
+_CODE_TO_COUNTRY = {
+    "W": "WALES",
+    "S": "SCOTLAND",
+}
+
+
+def add_private_rent_targets(
+    matrix: pd.DataFrame,
+    y: pd.DataFrame,
+    age_targets: pd.DataFrame,
+    *,
+    country: np.ndarray,
+    tenure_type: np.ndarray,
+    rent: np.ndarray,
+) -> None:
+    """Append Wales/Scotland private-rent count and amount targets.
+
+    Country totals are allocated across 2010 constituencies in proportion to
+    their official age-target population shares within each country.
+    """
+
+    constituency_population = age_targets.filter(like="age/").sum(axis=1)
+    constituency_country = age_targets["code"].str[0].map(_CODE_TO_COUNTRY)
+    private_renter = tenure_type == "RENT_PRIVATELY"
+
+    for country_name, target in _PRIVATE_RENT_TARGETS.items():
+        area_mask = constituency_country == country_name
+        country_population = constituency_population.where(area_mask, 0).sum()
+        if country_population <= 0:
+            raise ValueError(
+                f"No constituency population available for {country_name} housing targets"
+            )
+
+        share = np.where(area_mask, constituency_population / country_population, 0.0)
+        in_country_private_rent = (country == country_name) & private_renter
+        prefix = country_name.lower()
+
+        matrix[f"housing/{prefix}_private_renter_households"] = (
+            in_country_private_rent
+        ).astype(float)
+        matrix[f"housing/{prefix}_private_rent_amount"] = np.where(
+            in_country_private_rent,
+            rent,
+            0.0,
+        )
+
+        y[f"housing/{prefix}_private_renter_households"] = (
+            share * target["private_renter_households"]
+        )
+        y[f"housing/{prefix}_private_rent_amount"] = (
+            share * target["private_renter_households"] * target["annual_private_rent"]
+        )

policyengine_uk_data/datasets/local_areas/constituencies/loss.py

@@ -32,6 +32,9 @@
     get_constituency_uc_targets,
     get_constituency_uc_by_children_targets,
 )
+from policyengine_uk_data.datasets.local_areas.constituencies.devolved_housing import (
+    add_private_rent_targets,
+)
 
 
 def create_constituency_target_matrix(
@@ -114,6 +117,16 @@ def create_constituency_target_matrix(
     for col in uc_by_children.columns:
         y[col] = uc_by_children[col].values
 
+    # ── Wales/Scotland housing anchors ───────────────────────────────
+    add_private_rent_targets(
+        matrix,
+        y,
+        age_targets,
+        country=sim.calculate("country").values,
+        tenure_type=sim.calculate("tenure_type").values,
+        rent=sim.calculate("rent").values,
+    )
+
     # ── Boundary mapping (2010 → 2024) ────────────────────────────
     const_2024 = pd.read_csv(STORAGE_FOLDER / "constituencies_2024.csv")
 

policyengine_uk_data/targets/compute/households.py

@@ -60,28 +60,36 @@ def ft_hh(value):
 def compute_tenure(target, ctx) -> np.ndarray | None:
     """Compute dwelling count by tenure type."""
     _TENURE_MAP = {
-        "tenure_england_owned_outright": "OWNED_OUTRIGHT",
-        "tenure_england_owned_with_mortgage": "OWNED_WITH_MORTGAGE",
-        "tenure_england_rented_privately": "RENT_PRIVATELY",
-        "tenure_england_social_rent": [
-            "RENT_FROM_COUNCIL",
-            "RENT_FROM_HA",
-        ],
-        "tenure_england_total": None,
+        "tenure_england_owned_outright": ("OWNED_OUTRIGHT", "ENGLAND"),
+        "tenure_england_owned_with_mortgage": (
+            "OWNED_WITH_MORTGAGE",
+            "ENGLAND",
+        ),
+        "tenure_england_rented_privately": ("RENT_PRIVATELY", "ENGLAND"),
+        "tenure_england_social_rent": (
+            [
+                "RENT_FROM_COUNCIL",
+                "RENT_FROM_HA",
+            ],
+            "ENGLAND",
+        ),
+        "tenure_england_total": (None, "ENGLAND"),
     }
-    suffix = target.name.removeprefix("ons/")
-    pe_values = _TENURE_MAP.get(suffix)
-    if pe_values is None and suffix == "tenure_england_total":
-        return (ctx.country == "ENGLAND").astype(float)
-    if pe_values is None:
+    suffix = target.name.split("/", 1)[-1]
+    tenure_spec = _TENURE_MAP.get(suffix)
+    if tenure_spec is None:
         return None
 
+    pe_values, country = tenure_spec
+    if pe_values is None:
+        return (ctx.country == country).astype(float)
+
     tenure = ctx.sim.calculate("tenure_type", map_to="household").values
-    in_england = ctx.country == "ENGLAND"
+    in_country = ctx.country == country
     if isinstance(pe_values, list):
         match = np.zeros_like(tenure, dtype=bool)
         for v in pe_values:
             match = match | (tenure == v)
     else:
         match = tenure == pe_values
-    return (match & in_england).astype(float)
+    return (match & in_country).astype(float)

policyengine_uk_data/targets/compute/other.py

@@ -33,6 +33,7 @@ def compute_housing(target, ctx) -> np.ndarray:
         return ctx.pe("mortgage_capital_repayment") + ctx.pe(
             "mortgage_interest_repayment"
         )
+
     tenure = ctx.sim.calculate("tenure_type", map_to="household").values
     if name == "housing/rent_social":
         is_social = (tenure == "RENT_FROM_COUNCIL") | (tenure == "RENT_FROM_HA")

policyengine_uk_data/tests/test_devolved_housing_targets.py

@@ -0,0 +1,194 @@
+import numpy as np
+import pandas as pd
+
+from policyengine_uk_data.datasets.local_areas.constituencies.devolved_housing import (
+    _PRIVATE_RENT_TARGETS,
+    add_private_rent_targets,
+)
+from policyengine_uk_data.datasets.local_areas.constituencies import (
+    loss as constituency_loss,
+)
+
+
+def _age_targets():
+    return pd.DataFrame(
+        {
+            "code": ["W07000041", "W07000042", "S14000001", "S14000002"],
+            "name": ["W1", "W2", "S1", "S2"],
+            "age/0_10": [100, 300, 200, 200],
+            "age/10_20": [100, 300, 200, 200],
+        }
+    )
+
+
+def test_add_private_rent_targets_filters_matrix_to_country_private_renters():
+    matrix = pd.DataFrame()
+    y = pd.DataFrame()
+
+    add_private_rent_targets(
+        matrix,
+        y,
+        _age_targets(),
+        country=np.array(["WALES", "WALES", "SCOTLAND", "ENGLAND"]),
+        tenure_type=np.array(
+            ["RENT_PRIVATELY", "OWNED_OUTRIGHT", "RENT_PRIVATELY", "RENT_PRIVATELY"]
+        ),
+        rent=np.array([9_600.0, 0.0, 12_000.0, 15_000.0]),
+    )
+
+    np.testing.assert_array_equal(
+        matrix["housing/wales_private_renter_households"].values,
+        np.array([1.0, 0.0, 0.0, 0.0]),
+    )
+    np.testing.assert_array_equal(
+        matrix["housing/scotland_private_renter_households"].values,
+        np.array([0.0, 0.0, 1.0, 0.0]),
+    )
+    np.testing.assert_array_equal(
+        matrix["housing/wales_private_rent_amount"].values,
+        np.array([9_600.0, 0.0, 0.0, 0.0]),
+    )
+    np.testing.assert_array_equal(
+        matrix["housing/scotland_private_rent_amount"].values,
+        np.array([0.0, 0.0, 12_000.0, 0.0]),
+    )
+
+
+def test_add_private_rent_targets_allocate_country_totals_by_population_share():
+    matrix = pd.DataFrame()
+    y = pd.DataFrame()
+
+    add_private_rent_targets(
+        matrix,
+        y,
+        _age_targets(),
+        country=np.array(["WALES", "SCOTLAND"]),
+        tenure_type=np.array(["RENT_PRIVATELY", "RENT_PRIVATELY"]),
+        rent=np.array([9_600.0, 12_000.0]),
+    )
+
+    wales_shares = np.array([0.25, 0.75, 0.0, 0.0])
+    scotland_shares = np.array([0.0, 0.0, 0.5, 0.5])
+
+    np.testing.assert_allclose(
+        y["housing/wales_private_renter_households"].values,
+        wales_shares * _PRIVATE_RENT_TARGETS["WALES"]["private_renter_households"],
+    )
+    np.testing.assert_allclose(
+        y["housing/scotland_private_renter_households"].values,
+        scotland_shares
+        * _PRIVATE_RENT_TARGETS["SCOTLAND"]["private_renter_households"],
+    )
+    np.testing.assert_allclose(
+        y["housing/wales_private_rent_amount"].values.sum(),
+        _PRIVATE_RENT_TARGETS["WALES"]["private_renter_households"]
+        * _PRIVATE_RENT_TARGETS["WALES"]["annual_private_rent"],
+    )
+    np.testing.assert_allclose(
+        y["housing/scotland_private_rent_amount"].values.sum(),
+        _PRIVATE_RENT_TARGETS["SCOTLAND"]["private_renter_households"]
+        * _PRIVATE_RENT_TARGETS["SCOTLAND"]["annual_private_rent"],
+    )
+
+
+class _FakeDataset:
+    time_period = 2025
+
+
+class _FakeSim:
+    def __init__(self, *args, **kwargs):
+        self.default_calculation_period = 2025
+
+    def calculate(self, variable):
+        mapping = {
+            "self_employment_income": np.array([0.0, 0.0]),
+            "employment_income": np.array([0.0, 0.0]),
+            "income_tax": np.array([1.0, 1.0]),
+            "age": np.array([35, 35]),
+            "universal_credit": np.array([1.0, 1.0]),
+            "is_child": np.array([0.0, 0.0]),
+            "country": np.array(["WALES", "SCOTLAND"]),
+            "tenure_type": np.array(["RENT_PRIVATELY", "RENT_PRIVATELY"]),
+            "rent": np.array([9_600.0, 12_000.0]),
+        }
+        return type("Result", (), {"values": mapping[variable]})()
+
+    def map_result(self, values, source_entity, target_entity):
+        return np.asarray(values)
+
+
+def test_constituency_target_matrix_includes_devolved_housing_targets(monkeypatch):
+    age_targets = _age_targets().iloc[[0, 2]].reset_index(drop=True)
+    income_targets = pd.DataFrame(
+        {
+            "self_employment_income_amount": [1.0, 1.0],
+            "self_employment_income_count": [1.0, 1.0],
+            "employment_income_amount": [1.0, 1.0],
+            "employment_income_count": [1.0, 1.0],
+        }
+    )
+    national_income = pd.DataFrame(
+        {
+            "total_income_lower_bound": [12_570],
+            "total_income_upper_bound": [np.inf],
+            "self_employment_income_amount": [1.0],
+            "employment_income_amount": [1.0],
+        }
+    )
+    uc_by_children = pd.DataFrame(
+        {
+            "uc_hh_0_children": [1.0, 1.0],
+            "uc_hh_1_child": [0.0, 0.0],
+            "uc_hh_2_children": [0.0, 0.0],
+            "uc_hh_3plus_children": [0.0, 0.0],
+        }
+    )
+
+    monkeypatch.setattr(constituency_loss, "Microsimulation", _FakeSim)
+    monkeypatch.setattr(
+        constituency_loss, "get_constituency_income_targets", lambda: income_targets
+    )
+    monkeypatch.setattr(
+        constituency_loss,
+        "get_national_income_projections",
+        lambda year: national_income,
+    )
+    monkeypatch.setattr(
+        constituency_loss, "get_constituency_age_targets", lambda: age_targets
+    )
+    monkeypatch.setattr(constituency_loss, "get_uk_total_population", lambda year: 2.0)
+    monkeypatch.setattr(
+        constituency_loss,
+        "get_constituency_uc_targets",
+        lambda: pd.Series([1.0, 1.0]),
+    )
+    monkeypatch.setattr(
+        constituency_loss,
+        "get_constituency_uc_by_children_targets",
+        lambda: uc_by_children,
+    )
+    monkeypatch.setattr(constituency_loss, "mapping_matrix", np.eye(2))
+    monkeypatch.setattr(
+        constituency_loss.pd,
+        "read_csv",
+        lambda path: pd.DataFrame({"code": ["W07000041", "S14000001"]}),
+    )
+
+    matrix, y, country_mask = constituency_loss.create_constituency_target_matrix(
+        _FakeDataset()
+    )
+
+    assert "housing/wales_private_renter_households" in matrix.columns
+    assert "housing/scotland_private_rent_amount" in matrix.columns
+    np.testing.assert_allclose(
+        y["housing/wales_private_renter_households"].values,
+        np.array([_PRIVATE_RENT_TARGETS["WALES"]["private_renter_households"], 0.0]),
+    )
+    np.testing.assert_allclose(
+        y["housing/scotland_private_renter_households"].values,
+        np.array([0.0, _PRIVATE_RENT_TARGETS["SCOTLAND"]["private_renter_households"]]),
+    )
+    np.testing.assert_array_equal(
+        country_mask,
+        np.array([[1.0, 0.0], [0.0, 1.0]]),
+    )