From 070f55bb1b837a0801e474d90b73010a8b227c3a Mon Sep 17 00:00:00 2001 From: Nikhil Woodruff Date: Mon, 8 Dec 2025 11:10:41 +0000 Subject: [PATCH 1/6] Add housing implied admin targets --- policyengine_uk_data/utils/loss.py | 50 ++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/policyengine_uk_data/utils/loss.py b/policyengine_uk_data/utils/loss.py index 599008df..ca6cdfec 100644 --- a/policyengine_uk_data/utils/loss.py +++ b/policyengine_uk_data/utils/loss.py @@ -574,6 +574,56 @@ def pe_count(*variables): target_names.append("nts/households_two_plus_vehicles") target_values.append(total_households * NTS_TWO_PLUS_VEHICLE_RATE) + GROSS_INCOME_ESTIMATE = { + "private_renter": 350e9, + "owner_mortgage": 690e9, + "council_renter": 76e9, + "ha_renter": 134e9, + } + + HOUSING_COST_AS_INCOME_SHARE = { + "private_renter": 0.340, + "owner_mortgage": 0.191, + "council_renter": 0.276, + "ha_renter": 0.286, + } + + # Housing affordability targets + # Total mortgage payments (capital + interest) + mortgage_capital = pe("mortgage_capital_repayment") + mortgage_interest = pe("mortgage_interest_repayment") + total_mortgage = mortgage_capital + mortgage_interest + df["housing/total_mortgage"] = total_mortgage + target_names.append("housing/total_mortgage") + target_values.append( + GROSS_INCOME_ESTIMATE["owner_mortgage"] + / HOUSING_COST_AS_INCOME_SHARE["owner_mortgage"] + ) + + # Total rent by tenure type + rent = pe("rent") + tenure_type = sim.calculate("tenure_type", map_to="household").values + + df["housing/rent_private"] = rent * (tenure_type == "RENT_PRIVATELY") + target_names.append("housing/rent_private") + target_values.append( + GROSS_INCOME_ESTIMATE["private_renter"] + / HOUSING_COST_AS_INCOME_SHARE["private_renter"] + ) + + df["housing/rent_council"] = rent * (tenure_type == "RENT_FROM_COUNCIL") + target_names.append("housing/rent_council") + target_values.append( + GROSS_INCOME_ESTIMATE["council_renter"] + / HOUSING_COST_AS_INCOME_SHARE["council_renter"] + ) + df["housing/rent_ha"] = rent * (tenure_type == "RENT_FROM_HA") + target_names.append("housing/rent_ha") + target_values.append( + GROSS_INCOME_ESTIMATE["ha_renter"] + / HOUSING_COST_AS_INCOME_SHARE["ha_renter"] + ) + combined_targets = pd.concat( [ targets, From 67a4ac4d6935642c4ae573a69ecb1b94e0206e17 Mon Sep 17 00:00:00 2001 From: Nikhil Woodruff Date: Mon, 8 Dec 2025 11:29:12 +0000 Subject: [PATCH 2/6] Multiplier the wrong way around --- policyengine_uk_data/utils/loss.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/policyengine_uk_data/utils/loss.py b/policyengine_uk_data/utils/loss.py index ca6cdfec..2fe7bd04 100644 --- a/policyengine_uk_data/utils/loss.py +++ b/policyengine_uk_data/utils/loss.py @@ -597,7 +597,7 @@ def pe_count(*variables): target_names.append("housing/total_mortgage") target_values.append( GROSS_INCOME_ESTIMATE["owner_mortgage"] - / HOUSING_COST_AS_INCOME_SHARE["owner_mortgage"] + * HOUSING_COST_AS_INCOME_SHARE["owner_mortgage"] ) # Total rent by tenure type @@ -608,20 +608,20 @@ def pe_count(*variables): target_names.append("housing/rent_private") target_values.append( GROSS_INCOME_ESTIMATE["private_renter"] - / HOUSING_COST_AS_INCOME_SHARE["private_renter"] + * HOUSING_COST_AS_INCOME_SHARE["private_renter"] ) df["housing/rent_council"] = rent * (tenure_type == "RENT_FROM_COUNCIL") target_names.append("housing/rent_council") target_values.append( GROSS_INCOME_ESTIMATE["council_renter"] - / HOUSING_COST_AS_INCOME_SHARE["council_renter"] + * HOUSING_COST_AS_INCOME_SHARE["council_renter"] ) df["housing/rent_ha"] = rent * (tenure_type == "RENT_FROM_HA") target_names.append("housing/rent_ha") target_values.append( GROSS_INCOME_ESTIMATE["ha_renter"] - / HOUSING_COST_AS_INCOME_SHARE["ha_renter"] + * HOUSING_COST_AS_INCOME_SHARE["ha_renter"] ) combined_targets = pd.concat( From 37692a630c6ab93134965baebcd64c50f87b98dc Mon Sep 17 00:00:00 2001 From: Nikhil Woodruff Date: Mon, 8 Dec 2025 17:03:48 +0000 Subject: [PATCH 3/6] Add mortgage and private rent targets --- .../datasets/imputations/income.py | 11 +++++- policyengine_uk_data/utils/calibrate.py | 2 +- policyengine_uk_data/utils/loss.py | 36 +++---------------- 3 files changed, 16 insertions(+), 33 deletions(-) diff --git a/policyengine_uk_data/datasets/imputations/income.py b/policyengine_uk_data/datasets/imputations/income.py index 857c6eec..6849e4bf 100644 --- a/policyengine_uk_data/datasets/imputations/income.py +++ b/policyengine_uk_data/datasets/imputations/income.py @@ -153,13 +153,22 @@ def impute_over_incomes( DataFrame with imputed income components. """ dataset = dataset.copy() - input_df = Microsimulation(dataset=dataset).calculate_dataframe( + sim = Microsimulation(dataset=dataset) + input_df = sim.calculate_dataframe( ["age", "gender", "region"] ) + original_income_total = dataset.person[IMPUTATIONS].copy().sum().sum() output_df = model.predict(input_df) for column in output_variables: dataset.person[column] = output_df[column].fillna(0).values + + new_income_total = dataset.person[IMPUTATIONS].sum().sum() + adjustment_factor = new_income_total / original_income_total + # Adjust rent and mortgage interest and capital repayments proportionally + dataset.household["rent"] = dataset.household["rent"] * adjustment_factor + dataset.household["mortgage_interest_repayment"] = dataset.household["mortgage_interest_repayment"] * adjustment_factor + dataset.household["mortgage_capital_repayment"] = dataset.household["mortgage_capital_repayment"] * adjustment_factor return dataset diff --git a/policyengine_uk_data/utils/calibrate.py b/policyengine_uk_data/utils/calibrate.py index ba91b42b..43188976 100644 --- a/policyengine_uk_data/utils/calibrate.py +++ b/policyengine_uk_data/utils/calibrate.py @@ -15,7 +15,7 @@ def calibrate_local_areas( area_count: int, weight_file: str, dataset_key: str = "2025", - epochs: int = 256, + epochs: int = 512, excluded_training_targets=[], log_csv=None, verbose: bool = False, diff --git a/policyengine_uk_data/utils/loss.py b/policyengine_uk_data/utils/loss.py index 2fe7bd04..d1113486 100644 --- a/policyengine_uk_data/utils/loss.py +++ b/policyengine_uk_data/utils/loss.py @@ -441,8 +441,6 @@ def pe_count(*variables): target_names.append("hmrc/salary_sacrifice_contributions") target_values.append(SS_CONTRIBUTIONS_2024 * uprating_factor) - print(target_names[-4:], target_values[-4:]) - # Add two-child limit targets. child_is_affected = ( sim.map_result( @@ -574,18 +572,9 @@ def pe_count(*variables): target_names.append("nts/households_two_plus_vehicles") target_values.append(total_households * NTS_TWO_PLUS_VEHICLE_RATE) - GROSS_INCOME_ESTIMATE = { - "private_renter": 350e9, - "owner_mortgage": 690e9, - "council_renter": 76e9, - "ha_renter": 134e9, - } - - HOUSING_COST_AS_INCOME_SHARE = { - "private_renter": 0.340, - "owner_mortgage": 0.191, - "council_renter": 0.276, - "ha_renter": 0.286, + RENT_ESTIMATE = { + "private_renter": 1_400 * 12 * 4.7e6, # https://www.ons.gov.uk/economy/inflationandpriceindices/bulletins/privaterentandhousepricesuk/january2025 + "owner_mortgage": 1_100 * 12 * 7.5e6, } # Housing affordability targets @@ -596,8 +585,7 @@ def pe_count(*variables): df["housing/total_mortgage"] = total_mortgage target_names.append("housing/total_mortgage") target_values.append( - GROSS_INCOME_ESTIMATE["owner_mortgage"] - * HOUSING_COST_AS_INCOME_SHARE["owner_mortgage"] + RENT_ESTIMATE["owner_mortgage"] ) # Total rent by tenure type @@ -607,21 +595,7 @@ def pe_count(*variables): df["housing/rent_private"] = rent * (tenure_type == "RENT_PRIVATELY") target_names.append("housing/rent_private") target_values.append( - GROSS_INCOME_ESTIMATE["private_renter"] - * HOUSING_COST_AS_INCOME_SHARE["private_renter"] - ) - - df["housing/rent_council"] = rent * (tenure_type == "RENT_FROM_COUNCIL") - target_names.append("housing/rent_council") - target_values.append( - GROSS_INCOME_ESTIMATE["council_renter"] - * HOUSING_COST_AS_INCOME_SHARE["council_renter"] - ) - df["housing/rent_ha"] = rent * (tenure_type == "RENT_FROM_HA") - target_names.append("housing/rent_ha") - target_values.append( - GROSS_INCOME_ESTIMATE["ha_renter"] - * HOUSING_COST_AS_INCOME_SHARE["ha_renter"] + RENT_ESTIMATE["private_renter"] ) combined_targets = pd.concat( From aeb79279f42343a51f76c516ff6de82565ed67f5 Mon Sep 17 00:00:00 2001 From: Nikhil Woodruff Date: Mon, 8 Dec 2025 17:04:03 +0000 Subject: [PATCH 4/6] Format --- .../datasets/imputations/income.py | 14 ++++++++------ policyengine_uk_data/utils/loss.py | 12 +++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/policyengine_uk_data/datasets/imputations/income.py b/policyengine_uk_data/datasets/imputations/income.py index 6849e4bf..5f0639a7 100644 --- a/policyengine_uk_data/datasets/imputations/income.py +++ b/policyengine_uk_data/datasets/imputations/income.py @@ -154,21 +154,23 @@ def impute_over_incomes( """ dataset = dataset.copy() sim = Microsimulation(dataset=dataset) - input_df = sim.calculate_dataframe( - ["age", "gender", "region"] - ) + input_df = sim.calculate_dataframe(["age", "gender", "region"]) original_income_total = dataset.person[IMPUTATIONS].copy().sum().sum() output_df = model.predict(input_df) for column in output_variables: dataset.person[column] = output_df[column].fillna(0).values - + new_income_total = dataset.person[IMPUTATIONS].sum().sum() adjustment_factor = new_income_total / original_income_total # Adjust rent and mortgage interest and capital repayments proportionally dataset.household["rent"] = dataset.household["rent"] * adjustment_factor - dataset.household["mortgage_interest_repayment"] = dataset.household["mortgage_interest_repayment"] * adjustment_factor - dataset.household["mortgage_capital_repayment"] = dataset.household["mortgage_capital_repayment"] * adjustment_factor + dataset.household["mortgage_interest_repayment"] = ( + dataset.household["mortgage_interest_repayment"] * adjustment_factor + ) + dataset.household["mortgage_capital_repayment"] = ( + dataset.household["mortgage_capital_repayment"] * adjustment_factor + ) return dataset diff --git a/policyengine_uk_data/utils/loss.py b/policyengine_uk_data/utils/loss.py index d1113486..27d6c1da 100644 --- a/policyengine_uk_data/utils/loss.py +++ b/policyengine_uk_data/utils/loss.py @@ -573,7 +573,9 @@ def pe_count(*variables): target_values.append(total_households * NTS_TWO_PLUS_VEHICLE_RATE) RENT_ESTIMATE = { - "private_renter": 1_400 * 12 * 4.7e6, # https://www.ons.gov.uk/economy/inflationandpriceindices/bulletins/privaterentandhousepricesuk/january2025 + "private_renter": 1_400 + * 12 + * 4.7e6, # https://www.ons.gov.uk/economy/inflationandpriceindices/bulletins/privaterentandhousepricesuk/january2025 "owner_mortgage": 1_100 * 12 * 7.5e6, } @@ -584,9 +586,7 @@ def pe_count(*variables): total_mortgage = mortgage_capital + mortgage_interest df["housing/total_mortgage"] = total_mortgage target_names.append("housing/total_mortgage") - target_values.append( - RENT_ESTIMATE["owner_mortgage"] - ) + target_values.append(RENT_ESTIMATE["owner_mortgage"]) # Total rent by tenure type rent = pe("rent") @@ -594,9 +594,7 @@ def pe_count(*variables): df["housing/rent_private"] = rent * (tenure_type == "RENT_PRIVATELY") target_names.append("housing/rent_private") - target_values.append( - RENT_ESTIMATE["private_renter"] - ) + target_values.append(RENT_ESTIMATE["private_renter"]) combined_targets = pd.concat( [ From 039a1958819ddb4ed484ac927e972859571ca57a Mon Sep 17 00:00:00 2001 From: Nikhil Woodruff Date: Mon, 8 Dec 2025 17:04:31 +0000 Subject: [PATCH 5/6] Changelog --- changelog_entry.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/changelog_entry.yaml b/changelog_entry.yaml index e69de29b..8364a657 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -0,0 +1,4 @@ +- bump: patch + changes: + fixed: + - Added mortgage and private rent targets. From f6d1d598a85d8e53f7efaf0eedd5a8302b5a042d Mon Sep 17 00:00:00 2001 From: Nikhil Woodruff Date: Mon, 8 Dec 2025 17:47:07 +0000 Subject: [PATCH 6/6] Update reform tests --- policyengine_uk_data/tests/microsimulation/reforms_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/policyengine_uk_data/tests/microsimulation/reforms_config.yaml b/policyengine_uk_data/tests/microsimulation/reforms_config.yaml index c22a9550..0fc42ba3 100644 --- a/policyengine_uk_data/tests/microsimulation/reforms_config.yaml +++ b/policyengine_uk_data/tests/microsimulation/reforms_config.yaml @@ -16,7 +16,7 @@ reforms: parameters: gov.hmrc.child_benefit.amount.additional: 25 - name: Reduce Universal Credit taper rate to 20% - expected_impact: -27.2 + expected_impact: -30.9 parameters: gov.dwp.universal_credit.means_test.reduction_rate: 0.2 - name: Raise Class 1 main employee NICs rate to 10%