From 754505ec60c761f6bafd553d75da07aee4f8c9e9 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Sun, 26 Jul 2026 12:39:55 +0200 Subject: [PATCH 1/3] Net Scottish Water charges out of FRS council tax MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In Scotland, council tax bills are collected together with Scottish Water and sewerage charges, and the FRS CTANNUAL variable includes them. The FRS processing used CTANNUAL directly (and in the region/band/single imputation-table means), so modelled Scottish council tax included ~£500 per household of water charges that are also captured separately in water_and_sewerage_charges — a double count that put modelled Scottish council tax (~£4.0bn net, 2026) roughly 25% above Scottish Government receipts (~£3.1-3.3bn), while England was within a few per cent. Subtract the annualised Scottish water/sewerage amounts (csewamt + cwatamtd) from CTANNUAL for Scottish households, in both the reported values and the imputation table, floored at zero. Ref PolicyEngine/policyengine-uk-data#448 Co-Authored-By: Claude Fable 5 --- changelog.d/scottish-ct-water-charges.md | 1 + policyengine_uk_data/datasets/frs.py | 28 ++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 changelog.d/scottish-ct-water-charges.md diff --git a/changelog.d/scottish-ct-water-charges.md b/changelog.d/scottish-ct-water-charges.md new file mode 100644 index 00000000..1223fb62 --- /dev/null +++ b/changelog.d/scottish-ct-water-charges.md @@ -0,0 +1 @@ +Net Scottish Water and sewerage charges out of FRS-reported council tax (CTANNUAL includes them in Scotland), fixing a ~£500-per-household double count that overstated Scottish council tax by roughly 25% relative to Scottish Government receipts; the charges remain captured in water_and_sewerage_charges. diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index c717d97e..98bc8c37 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -817,6 +817,30 @@ def determine_education_level(fted_val, typeed2_val, age_val): # Impute Council Tax + # In Scotland, council tax bills are collected together with Scottish + # Water and sewerage charges, and the FRS CTANNUAL variable includes + # them. Net them off (they are weekly variables; CTANNUAL is annual) so + # council_tax is tax only: the water charges are already captured + # separately in water_and_sewerage_charges, so leaving them in both + # double-counts them and overstates Scottish council tax by roughly + # £500 per household (~25% of the Scottish total). + SCOTLAND_GVTREGNO = 12 + scottish_water_annual = pd.Series( + np.where( + household.gvtregno == SCOTLAND_GVTREGNO, + ( + np.maximum(household.csewamt.fillna(0), 0) + + np.maximum(household.cwatamtd.fillna(0), 0) + ) + * (365.25 / 7), + 0, + ), + index=household.index, + ) + ctannual_tax_only = np.maximum( + household.ctannual - scottish_water_annual, 0 + ) + # Only ~25% of household report Council Tax bills - use # these to build a model to impute missing values CT_valid = household.ctannual > 0 @@ -826,7 +850,7 @@ def determine_education_level(fted_val, typeed2_val, age_val): region = household.gvtregno[CT_valid] band = household.ctband[CT_valid] single_person = (household.adulth == 1)[CT_valid] - ctannual = household.ctannual[CT_valid] + ctannual = ctannual_tax_only[CT_valid] # Build the table ct_mean = ctannual.groupby([region, band, single_person], dropna=False).mean() @@ -851,7 +875,7 @@ def determine_education_level(fted_val, typeed2_val, age_val): # uses -1 for missing values (household.ctannual < 0) | household.ctannual.isna(), np.maximum(ct_imputed, 0).values, - household.ctannual, + ctannual_tax_only, ) ) pe_household["council_tax"] = council_tax.fillna(0) From 7df3e4b09b1964c717506c1f9a5f1f8aca89acba Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Sun, 26 Jul 2026 12:57:31 +0200 Subject: [PATCH 2/3] Format Scottish council tax adjustment --- policyengine_uk_data/datasets/frs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 98bc8c37..df4cb3a3 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -837,9 +837,7 @@ def determine_education_level(fted_val, typeed2_val, age_val): ), index=household.index, ) - ctannual_tax_only = np.maximum( - household.ctannual - scottish_water_annual, 0 - ) + ctannual_tax_only = np.maximum(household.ctannual - scottish_water_annual, 0) # Only ~25% of household report Council Tax bills - use # these to build a model to impute missing values From 9ffe92e0038c9ceabc2d2a38aa4c2f54f5598977 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Sun, 26 Jul 2026 13:00:35 +0200 Subject: [PATCH 3/3] Remove macOS metadata artifact --- .gitignore | 3 +++ .../storage/spi_2022_23/__MACOSX/._README.md | Bin 163 -> 0 bytes 2 files changed, 3 insertions(+) delete mode 100644 policyengine_uk_data/storage/spi_2022_23/__MACOSX/._README.md diff --git a/.gitignore b/.gitignore index 1e629c60..9a741bc4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ *.egg-info **/__pycache__ **/.DS_STORE +**/.DS_Store +**/__MACOSX +**/._* **/*.h5 **/*.csv **/*.zip diff --git a/policyengine_uk_data/storage/spi_2022_23/__MACOSX/._README.md b/policyengine_uk_data/storage/spi_2022_23/__MACOSX/._README.md deleted file mode 100644 index 3fdac2e382b91375b49528950c678576e329a19c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 163 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDI}aUl?c_=|y<2;dkJ5(HHS(lG;wxzV&S oBE&_L^K