Fix for missing IMD data in downloads

drkane · drkane · commit 6134ff6b65a0 · 2025-11-17T11:05:36.000Z
diff --git a/findthatpostcode/blueprints/points.py b/findthatpostcode/blueprints/points.py
@@ -36,5 +36,6 @@ def get(latlon):
             filetype,
             "postcode.html.j2",
             point=result,
+            stats=result.relationships["nearest_postcode"].get_stats(),
         )
     return return_result(result, filetype, "postcode.html.j2")
diff --git a/findthatpostcode/blueprints/postcodes.py b/findthatpostcode/blueprints/postcodes.py
@@ -5,7 +5,12 @@
 from findthatpostcode.blueprints.utils import return_result
 from findthatpostcode.controllers.postcodes import Postcode
 from findthatpostcode.db import get_db
-from findthatpostcode.metadata import STATS_FIELDS
+from findthatpostcode.metadata import (
+    OAC11_CODE,
+    RU11IND_CODES,
+    RUC21_CODES,
+    STATS_FIELDS,
+)
 
 bp = Blueprint("postcodes", __name__, url_prefix="/postcodes")
 
@@ -23,7 +28,7 @@ def postcode_redirect():
 def get_postcode(postcode, filetype="json"):
     es = get_db()
     result = Postcode.get_from_es(postcode, es)
-    return return_result(result, filetype, "postcode.html.j2")
+    return return_result(result, filetype, "postcode.html.j2", stats=result.get_stats())
 
 
 @bp.route("/hash/<hash>")
@@ -40,7 +45,7 @@ def multi_hash():
     return jsonify({"data": get_postcode_by_hash(hashes, fields)})
 
 
-def get_postcode_by_hash(hashes, fields):
+def get_postcode_by_hash(hashes: str | list[str], fields: list[str]):
     es = get_db()
 
     if not isinstance(hashes, list):
@@ -59,17 +64,19 @@ def get_postcode_by_hash(hashes, fields):
         )
 
     name_fields = [i.replace("_name", "") for i in fields if i.endswith("_name")]
-    extra_fields = []
-    stats = [i for i in STATS_FIELDS if i[0] in fields]
+    stats_fields = []
+    stats = [field for field in STATS_FIELDS if field.id in fields]
     if stats:
-        extra_fields.append("lsoa11")
+        for field in stats:
+            if field.area not in stats_fields:
+                stats_fields.append(field.area)
 
     results = list(
         scan(
             es,
             index="geo_postcode",
             query={"query": {"bool": {"should": query}}},
-            _source_includes=fields + name_fields + extra_fields,
+            _source_includes=fields + name_fields + stats_fields,
         )
     )
     areas = scan(
@@ -81,39 +88,45 @@ def get_postcode_by_hash(hashes, fields):
     areanames = {i["_id"]: i["_source"].get("name") for i in areas}
 
     def get_names(data):
-        return {
-            i: areanames.get(data.get(i.replace("_name", "")))
-            for i in fields
-            if i.endswith("_name")
-        }
-
-    lsoas = {}
-
-    def get_stats(data):
-        lsoa = data.get("lsoa11")
-        if not lsoa or not stats or lsoa not in lsoas:
-            return {}
-        return {i[0]: dig_get(lsoas[lsoa], i[3]) for i in stats}
+        names = {}
+        for i in name_fields:
+            names[f"{i}_name"] = None
+            if i == "oac11":
+                oac_name = OAC11_CODE.get(data.get(i))
+                if oac_name:
+                    names[f"{i}_name"] = " > ".join(oac_name)
+            elif i == "ru11ind":
+                names[f"{i}_name"] = RU11IND_CODES.get(data.get(i))
+            elif i == "ruc21":
+                names[f"{i}_name"] = RUC21_CODES.get(data.get(i))
+            else:
+                names[f"{i}_name"] = areanames.get(data.get(i))
+        return names
+
+    def get_stats(data, lsoas):
+        result = {}
+        for field in stats:
+            lsoa_code = data.get(field.area)
+            if not lsoa_code or not stats or lsoa_code not in lsoas:
+                continue
+            result[field.id] = dig_get(lsoas[lsoa_code], field.location)
+        return result
 
     if results:
+        lsoas = {}
         if stats:
+            lsoas_to_get = set()
+            for r in results:
+                for i in stats_fields:
+                    if r.get("_source", {}).get(i):
+                        lsoas_to_get.add(r.get("_source", {}).get(i))
             lsoas = {
                 i["_id"]: i["_source"]
                 for i in scan(
                     es,
                     index="geo_area",
-                    query={
-                        "query": {
-                            "terms": {
-                                "_id": [
-                                    r.get("_source", {}).get("lsoa11")
-                                    for r in results
-                                    if r.get("_source", {}).get("lsoa11")
-                                ]
-                            }
-                        }
-                    },
-                    _source_includes=[i[3] for i in stats],
+                    query={"query": {"terms": {"_id": list(lsoas_to_get)}}},
+                    _source_includes=[i.location for i in stats],
                 )
             }
 
@@ -122,7 +135,7 @@ def get_stats(data):
                 "id": r["_id"],
                 **r["_source"],
                 **get_names(r["_source"]),
-                **get_stats(r["_source"]),
+                **get_stats(r["_source"], lsoas),
             }
             for r in results
         ]
diff --git a/findthatpostcode/controllers/postcodes.py b/findthatpostcode/controllers/postcodes.py
@@ -1,10 +1,18 @@
 import re
 from datetime import datetime
 
+from dictlib import dig_get
+
 from findthatpostcode.controllers.areas import Area
 from findthatpostcode.controllers.controller import Controller
 from findthatpostcode.controllers.places import Place
-from findthatpostcode.metadata import OAC11_CODE, RU11IND_CODES, RUC21_CODES
+from findthatpostcode.metadata import (
+    OAC11_CODE,
+    OTHER_CODES,
+    RU11IND_CODES,
+    RUC21_CODES,
+    STATS_FIELDS,
+)
 
 
 class Postcode(Controller):
@@ -174,3 +182,19 @@ def toJSON(self, role="top"):
                 json[0]["attributes"][i] = json[0]["attributes"][i].strftime("%Y-%m-%d")
 
         return json
+
+    def get_stats(self):
+        stats_block = {}
+        for field in STATS_FIELDS:
+            field_type = field.id.removesuffix("_rank").removesuffix("_decile")
+            if self.attributes.get("ctry") is not None:
+                stats_block[f"{field_type}_total"] = OTHER_CODES.get("imd", {}).get(
+                    f"{self.attributes['ctry']}-{field_type}"
+                )
+            if self.found and field.area in self.attributes:
+                lsoa_code = self.attributes.get(field.area)
+                for area in self.relationships.get("areas", []):
+                    if area.id == lsoa_code:
+                        stats_block[field.id] = dig_get(area.attributes, field.location)
+                        break
+        return stats_block
diff --git a/findthatpostcode/metadata.py b/findthatpostcode/metadata.py
@@ -1,6 +1,8 @@
 import json
 import os
 
+from findthatpostcode.utils import Field
+
 with open(
     os.path.join(os.path.dirname(os.path.realpath(__file__)), "areatypes.json")
 ) as a:
@@ -67,55 +69,55 @@
 
 DEFAULT_UPLOAD_FIELDS = ["latlng", "laua", "laua_name", "rgn", "rgn_name"]
 BASIC_UPLOAD_FIELDS = [
-    ("latlng", "Latitude / Longitude", False),
-    ("estnrth", "OS Easting / Northing", False),
-    ("pcds", "Standardised postcode", False),
-    ("oac11", "2011 Output Area Classification (OAC)", True),
-    ("ru11ind", "2011 Census rural-urban classification", True),
-    ("ruc21", "2021 Census rural-urban classification", True),
+    Field(id="latlng", name="Latitude / Longitude", has_name=False),
+    Field(id="estnrth", name="OS Easting / Northing", has_name=False),
+    Field(id="pcds", name="Standardised postcode", has_name=False),
+    Field(id="oac11", name="2011 Output Area Classification (OAC)", has_name=True),
+    Field(id="ru11ind", name="2011 Census rural-urban classification", has_name=True),
+    Field(id="ruc21", name="2021 Census rural-urban classification", has_name=True),
 ]
 STATS_FIELDS = [
-    (
-        "imd2025_rank",
-        "Index of multiple deprivation (2025) rank",
-        False,
-        "stats.imd2025.imd_rank",
+    Field(
+        id="imd2025_rank",
+        name="Index of multiple deprivation (2025) rank",
+        location="stats.imd2025.imd_rank",
+        area="lsoa21",
     ),
-    (
-        "imd2025_decile",
-        "Index of multiple deprivation (2025) decile",
-        False,
-        "stats.imd2025.imd_decile",
+    Field(
+        id="imd2025_decile",
+        name="Index of multiple deprivation (2025) decile",
+        location="stats.imd2025.imd_decile",
+        area="lsoa21",
     ),
-    (
-        "imd2019_rank",
-        "Index of multiple deprivation (2019) rank",
-        False,
-        "stats.imd2019.imd_rank",
+    Field(
+        id="imd2019_rank",
+        name="Index of multiple deprivation (2019) rank",
+        location="stats.imd2019.imd_rank",
+        area="lsoa11",
     ),
-    (
-        "imd2019_decile",
-        "Index of multiple deprivation (2019) decile",
-        False,
-        "stats.imd2019.imd_decile",
+    Field(
+        id="imd2019_decile",
+        name="Index of multiple deprivation (2019) decile",
+        location="stats.imd2019.imd_decile",
+        area="lsoa11",
     ),
-    (
-        "imd2015_rank",
-        "Index of multiple deprivation (2015) rank",
-        False,
-        "stats.imd2015.imd_rank",
+    Field(
+        id="imd2015_rank",
+        name="Index of multiple deprivation (2015) rank",
+        location="stats.imd2015.imd_rank",
+        area="lsoa11",
     ),
-    (
-        "imd2015_decile",
-        "Index of multiple deprivation (2015) decile",
-        False,
-        "stats.imd2015.imd_decile",
+    Field(
+        id="imd2015_decile",
+        name="Index of multiple deprivation (2015) decile",
+        location="stats.imd2015.imd_decile",
+        area="lsoa11",
     ),
-    # (
-    #   "popn",
-    #   "Total population (2015)",
-    #   False,
-    #   "stats.population2015.population_total"
+    # Field(
+    #   id="popn2015",
+    #   name="Total population (2015)",
+    #   location="stats.population2015.population_total",
+    #   area="lsoa11",
     # ),
 ]
 
diff --git a/findthatpostcode/templates/addtocsv.html.j2 b/findthatpostcode/templates/addtocsv.html.j2
@@ -90,13 +90,13 @@
             </tr>
             {% for b in basic_fields %}
             <tr>
-              <td class="ph2">{{b[1]}}</td>
-              <td class="ph2 tc"><input type="checkbox" autocomplete="off" name="fields" value="{{b[0]}}"
-                  {% if b[0] in default_fields %} checked="checked" {% endif %}></td>
+              <td class="ph2">{{ b.name }}</td>
+              <td class="ph2 tc"><input type="checkbox" autocomplete="off" name="fields" value="{{ b.id }}"
+                  {% if b.id in default_fields %} checked="checked" {% endif %}></td>
               <td class="ph2 tc">
-                {% if b[2] %}
-                <input type="checkbox" autocomplete="off" name="fields" value="{{b[0]}}_name"
-                  {% if '%s_name' % b[0] in default_fields %} checked="checked" {% endif %}>
+                {% if b.has_name %}
+                <input type="checkbox" autocomplete="off" name="fields" value="{{ b.id }}_name"
+                  {% if (b.id + '_name') in default_fields %} checked="checked" {% endif %}>
                 {% endif %}
               </td>
             </tr>
@@ -108,9 +108,9 @@
             </tr>
             {% for b in stats_fields %}
             <tr>
-              <td class="ph2">{{b[1]}}</td>
-              <td class="ph2 tc"><input type="checkbox" autocomplete="off" name="fields" value="{{b[0]}}"
-                  {% if b[0] in default_fields %} checked="checked" {% endif %}></td>
+              <td class="ph2">{{ b.name }}</td>
+              <td class="ph2 tc"><input type="checkbox" autocomplete="off" name="fields" value="{{ b.id }}"
+                  {% if b.id in default_fields %} checked="checked" {% endif %}></td>
               <td class="ph2 tc">
               </td>
             </tr>
diff --git a/findthatpostcode/templates/postcode.html.j2 b/findthatpostcode/templates/postcode.html.j2
@@ -111,31 +111,26 @@
     {% endcall %}
     {% endif %}
 
-    {% for a in result.relationships.areas if a.attributes.stats and a.attributes.stats.imd2025 %}
-    {% if loop.first %}
-    {% set rank = a.attributes.stats.imd2025.imd_rank %}
-    {% set total_25 = other_codes.imd[result.attributes.ctry + "-imd2025"] %}
-    {% set total_19 = other_codes.imd[result.attributes.ctry + "-imd2019"] %}
-    {% set total_15 = other_codes.imd[result.attributes.ctry + "-imd2015"] %}
+    {% if stats.imd2025_rank is not none %}
     {% call info_block("Index of multiple deprivation (2025)") %}
-    <p><strong>{{ "{:,.0f}".format(rank) }}</strong> out of {{ "{:,.0f}".format(total_25) }} lower super output areas in
+    <p><strong>{{ "{:,.0f}".format(stats.imd2025_rank) }}</strong> out of {{ "{:,.0f}".format(stats.imd2025_total) }}
+      lower super output areas in
       {{ result.attributes.ctry_name }} (where 1 is the most deprived LSOA).</p>
-    <p><strong>{{ "{:.0%}".format( rank|float / total_25|float ) }}</strong> of LSOAs in
+    <p><strong>{{ "{:.0%}".format( stats.imd2025_rank|float / stats.imd2025_total|float ) }}</strong> of LSOAs in
       {{ result.attributes.ctry_name }}
       are more deprived than this one.</p>
-    {% if a.attributes.stats.imd2019 %}
+    {% if stats.imd2019_rank %}
     <p>In <strong>2019</strong>
-      {{ "{:.0%}".format( a.attributes.stats.imd2019.imd_rank|float / total_19|float ) }}</strong> of LSOAs in
+      {{ "{:.0%}".format( stats.imd2019_rank|float / stats.imd2019_total|float ) }}</strong> of LSOAs in
       {{ result.attributes.ctry.name }} were more deprived than this one.</p>
     {% endif %}
-    {% if a.attributes.stats.imd2015 %}
+    {% if stats.imd2015_rank %}
     <p>In <strong>2015</strong>
-      {{ "{:.0%}".format( a.attributes.stats.imd2015.imd_rank|float / total_15|float ) }}</strong> of LSOAs in
+      {{ "{:.0%}".format( stats.imd2015_rank|float / stats.imd2015_total|float ) }}</strong> of LSOAs in
       {{ result.attributes.ctry.name }} were more deprived than this one.</p>
     {% endif %}
     {% endcall %}
     {% endif %}
-    {% endfor %}
   </div>
 
   {% for i in key_area_types[1:] %}
diff --git a/findthatpostcode/utils.py b/findthatpostcode/utils.py
@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class Field:
+    id: str
+    name: str
+    location: str | None = None
+    has_name: bool = False
+    area: str | None = None

Original file line number	Diff line number	Diff line change
`@@ -36,5 +36,6 @@ def get(latlon):`
`36`	`36`	`filetype,`
`37`	`37`	`"postcode.html.j2",`
`38`	`38`	`point=result,`
	`39`	`+ stats=result.relationships["nearest_postcode"].get_stats(),`
`39`	`40`	`)`
`40`	`41`	`return return_result(result, filetype, "postcode.html.j2")`