Congressional District-Level Calibration with Geo Stacking Methodology

.github/workflows/validate_district_calibration.yml

@@ -0,0 +1,104 @@
+name: Validate District-Level Calibration
+
+on:
+  push:
+    branches:
+      - new-cd-var
+  workflow_dispatch:
+    inputs:
+      gcs_date:
+        description: 'GCS date prefix (e.g., 2025-10-22-1721)'
+        required: true
+        type: string
+
+jobs:
+  validate-and-upload:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Install dependencies
+        run: uv pip install -e .[dev] --system
+
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: "projects/322898545428/locations/global/workloadIdentityPools/policyengine-research-id-pool/providers/prod-github-provider"
+          service_account: "policyengine-research@policyengine-research.iam.gserviceaccount.com"
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+
+      - name: Download weights from GCS
+        run: |
+          GCS_DATE="${{ inputs.gcs_date || '2025-10-22-1721' }}"
+          echo "Downloading weights from gs://policyengine-calibration/$GCS_DATE/outputs/"
+          mkdir -p policyengine_us_data/storage/calibration
+          gsutil ls gs://policyengine-calibration/$GCS_DATE/outputs/**/w_cd.npy | head -1 | xargs -I {} gsutil cp {} policyengine_us_data/storage/calibration/w_cd.npy
+          echo "Downloaded w_cd.npy"
+
+      - name: Download prerequisite datasets
+        run: |
+          GCS_DATE="${{ inputs.gcs_date || '2025-10-22-1721' }}"
+          echo "Downloading stratified dataset and database from calibration run..."
+          mkdir -p policyengine_us_data/storage
+          gsutil cp gs://policyengine-calibration/$GCS_DATE/inputs/stratified_extended_cps_2023.h5 policyengine_us_data/storage/
+          gsutil cp gs://policyengine-calibration/$GCS_DATE/inputs/policy_data.db policyengine_us_data/storage/
+
+      - name: Verify downloaded files
+        run: |
+          echo "Verifying downloaded files exist..."
+          if [ ! -f policyengine_us_data/storage/stratified_extended_cps_2023.h5 ]; then
+            echo "ERROR: stratified_extended_cps_2023.h5 not found"
+            exit 1
+          fi
+          if [ ! -f policyengine_us_data/storage/policy_data.db ]; then
+            echo "ERROR: policy_data.db not found"
+            exit 1
+          fi
+          echo "All required files present:"
+          ls -lh policyengine_us_data/storage/stratified_extended_cps_2023.h5
+          ls -lh policyengine_us_data/storage/policy_data.db
+
+      - name: Create state files
+        run: |
+          echo "Creating state-level .h5 files..."
+          python -m policyengine_us_data.datasets.cps.geo_stacking_calibration.create_sparse_cd_stacked \
+            --weights-path policyengine_us_data/storage/calibration/w_cd.npy \
+            --dataset-path policyengine_us_data/storage/stratified_extended_cps_2023.h5 \
+            --db-path policyengine_us_data/storage/policy_data.db \
+            --output-dir policyengine_us_data/storage/cd_states
+
+      - name: Run district-level validation tests
+        run: |
+          echo "Running validation tests..."
+          pytest -m "district_level_validation" -v
+
+      - name: Upload state files to GCS
+        if: success()
+        run: |
+          GCS_DATE="${{ inputs.gcs_date || '2025-10-22-1721' }}"
+          echo "Tests passed! Uploading state files to GCS..."
+          gsutil -m cp policyengine_us_data/storage/cd_states/*.h5 gs://policyengine-calibration/$GCS_DATE/state_files/
+          gsutil -m cp policyengine_us_data/storage/cd_states/*_household_mapping.csv gs://policyengine-calibration/$GCS_DATE/state_files/
+          echo ""
+          echo "✅ State files uploaded to gs://policyengine-calibration/$GCS_DATE/state_files/"
+
+      - name: Report validation failure
+        if: failure()
+        run: |
+          echo "❌ District-level calibration validation FAILED"
+          echo "Check the test output above for details"
+          echo "State files were NOT uploaded to GCS"
+          exit 1

.gitignore

@@ -24,3 +24,7 @@ node_modules
 !policyengine_us_data/storage/social_security_aux.csv
 !policyengine_us_data/storage/SSPopJul_TR2024.csv
 docs/.ipynb_checkpoints/
+
+# Geo-stacking pipeline outputs
+policyengine_us_data/storage/calibration/
+policyengine_us_data/storage/cd_states/

Makefile

@@ -56,6 +56,7 @@ documentation-dev:
 database:
 	python policyengine_us_data/db/create_database_tables.py
 	python policyengine_us_data/db/create_initial_strata.py
+	python policyengine_us_data/db/etl_national_targets.py
 	python policyengine_us_data/db/etl_age.py
 	python policyengine_us_data/db/etl_medicaid.py
 	python policyengine_us_data/db/etl_snap.py
@@ -74,9 +75,79 @@ data:
 	mv policyengine_us_data/storage/enhanced_cps_2024.h5 policyengine_us_data/storage/dense_enhanced_cps_2024.h5
 	cp policyengine_us_data/storage/sparse_enhanced_cps_2024.h5 policyengine_us_data/storage/enhanced_cps_2024.h5
 
+data-geo: data
+	GEO_STACKING=true python policyengine_us_data/datasets/cps/cps.py
+	GEO_STACKING=true python policyengine_us_data/datasets/puf/puf.py
+	GEO_STACKING_MODE=true python policyengine_us_data/datasets/cps/extended_cps.py
+	python policyengine_us_data/datasets/cps/geo_stacking_calibration/create_stratified_cps.py 10000
+
+calibration-package: data-geo
+	python policyengine_us_data/datasets/cps/geo_stacking_calibration/create_calibration_package.py \
+		--db-path policyengine_us_data/storage/policy_data.db \
+		--dataset-uri policyengine_us_data/storage/stratified_extended_cps_2023.h5 \
+		--mode Stratified \
+		--local-output policyengine_us_data/storage/calibration
+
+optimize-weights-local: calibration-package
+	python policyengine_us_data/datasets/cps/geo_stacking_calibration/optimize_weights.py \
+		--input-dir policyengine_us_data/storage/calibration \
+		--output-dir policyengine_us_data/storage/calibration \
+		--total-epochs 100 \
+		--device cpu
+
+create-state-files: optimize-weights-local
+	python -m policyengine_us_data.datasets.cps.geo_stacking_calibration.create_sparse_cd_stacked \
+		--weights-path policyengine_us_data/storage/calibration/w_cd.npy \
+		--dataset-path policyengine_us_data/storage/stratified_extended_cps_2023.h5 \
+		--db-path policyengine_us_data/storage/policy_data.db \
+		--output-dir policyengine_us_data/storage/cd_states
+
+upload-calibration-package: calibration-package
+	$(eval GCS_DATE := $(shell date +%Y-%m-%d-%H%M))  # For bash: GCS_DATE=$$(date +%Y-%m-%d-%H%M)
+	python policyengine_us_data/datasets/cps/geo_stacking_calibration/create_calibration_package.py \
+		--db-path policyengine_us_data/storage/policy_data.db \
+		--dataset-uri policyengine_us_data/storage/stratified_extended_cps_2023.h5 \
+		--mode Stratified \
+		--gcs-bucket policyengine-calibration \
+		--gcs-date $(GCS_DATE)
+	@echo "Uploading dataset and database to GCS inputs..."
+	gsutil cp policyengine_us_data/storage/stratified_extended_cps_2023.h5 gs://policyengine-calibration/$(GCS_DATE)/inputs/
+	gsutil cp policyengine_us_data/storage/policy_data.db gs://policyengine-calibration/$(GCS_DATE)/inputs/
+	@echo ""
+	@echo "Calibration package uploaded to GCS"
+	@echo "Date prefix: $(GCS_DATE)"
+	@echo ""
+	@echo "To submit GCP batch job, update batch_pipeline/config.env:"
+	@echo "  INPUT_PATH=$(GCS_DATE)/inputs"
+	@echo "  OUTPUT_PATH=$(GCS_DATE)/outputs"
+
+optimize-weights-gcp:
+	@echo "Submitting Cloud Batch job for weight optimization..."
+	@echo "Make sure you've run 'make upload-calibration-package' first"
+	@echo "and updated batch_pipeline/config.env with the correct paths"
+	@echo ""
+	cd policyengine_us_data/datasets/cps/geo_stacking_calibration/batch_pipeline && ./submit_batch_job.sh
+
+download-weights-from-gcs:
+	@echo "Downloading weights from GCS..."
+	rm -f policyengine_us_data/storage/calibration/w_cd.npy
+	@read -p "Enter GCS date prefix (e.g., 2025-10-22-1630): " gcs_date; \
+	gsutil ls gs://policyengine-calibration/$$gcs_date/outputs/**/w_cd.npy | head -1 | xargs -I {} gsutil cp {} policyengine_us_data/storage/calibration/w_cd.npy && \
+	gsutil ls gs://policyengine-calibration/$$gcs_date/outputs/**/w_cd_*.npy | xargs -I {} gsutil cp {} policyengine_us_data/storage/calibration/ && \
+	echo "Weights downloaded successfully"
+
+upload-state-files-to-gcs:
+	@echo "Uploading state files to GCS..."
+	@read -p "Enter GCS date prefix (e.g., 2025-10-22-1721): " gcs_date; \
+	gsutil -m cp policyengine_us_data/storage/cd_states/*.h5 gs://policyengine-calibration/$$gcs_date/state_files/ && \
+	gsutil -m cp policyengine_us_data/storage/cd_states/*_household_mapping.csv gs://policyengine-calibration/$$gcs_date/state_files/ && \
+	echo "" && \
+	echo "State files uploaded to gs://policyengine-calibration/$$gcs_date/state_files/"
+
 clean:
 	rm -f policyengine_us_data/storage/*.h5
 	rm -f policyengine_us_data/storage/*.db
+	rm -f policyengine_us_data/storage/*.pkl
 	git clean -fX -- '*.csv'
 	rm -rf policyengine_us_data/docs/_build
 

changelog_entry.yaml

@@ -0,0 +1,9 @@
+- bump: minor
+  changes:
+    added:
+    - Targets database infrastructure for geo-stacking calibration
+    - Congressional district level estimation capability
+    - Geo-stacking calibration utilities and modeling functionality
+    - GEO_STACKING environment variable for specialized data pipeline
+    - Hierarchical validation for calibration targets
+    - Holdout validation framework for geo-stacking models