Improved CI/CD pipeline, added dependencies, and other miscellaneous improvements (#211)

* Enable workflow_dispatch for pr_code_changes

* trying to get action to run

* changing workflow to require Python 3.12

* adding MinimalInstallCheck

* pyproject.toml

* Readme updates and Windows smoke test

* adding back the actions space

* comment out __file__ override

* dynamically getting yaml file. Removing commented code

* workflow updates

* Token work, matrix OS testing

* lints

* environment variables are put back

* Windows encodings fixed for documentation. Github code reset

* updated clean function and docs

* Makefile changes

* getting rid of workflow comments

* added comma

* taking out 3.10

* getting rid of growth factors

* trying a different dependency naming

Author: baogorek

.github/workflows/code_changes.yaml

@@ -38,7 +38,6 @@ jobs:
             with:
               workload_identity_provider: "projects/322898545428/locations/global/workloadIdentityPools/policyengine-research-id-pool/providers/prod-github-provider"
               service_account: "policyengine-research@policyengine-research.iam.gserviceaccount.com"
-              
           - name: Install package
             run: uv pip install -e .[dev] --system
           - name: Download data inputs
@@ -57,4 +56,4 @@ jobs:
             env:
               HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
           - name: Test documentation builds
-            run: make documentation
+            run: make documentation

.github/workflows/pr_code_changes.yaml

@@ -9,6 +9,7 @@ on:
     paths:
       - policyengine_us_data/**
       - tests/**
+      - .github/workflows/**
 
 jobs:
   Lint:
@@ -19,32 +20,68 @@ jobs:
           uses: "lgeiger/black-action@master"
           with:
               args: ". -l 79 --check"
+
+  SmokeTestForMultipleVersions:
+    name: Test Minimal Install & Import (${{ matrix.os }}, py${{ matrix.python-version }})
+    runs-on: ${{ matrix.os }}
+    needs: Lint
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        python-version: ['3.11', '3.12']
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install package ONLY (no dev deps)
+        run: python -m pip install .
+
+      - name: Test basic import
+        run: python -c "import policyengine_us_data; print('Minimal import OK')"
+
+      - name: Test specific core import
+        run: python -c "from policyengine_core.data import Dataset; print('Core import OK')"
+
   Test:
-      runs-on: ubuntu-latest
+      runs-on: ${{ matrix.os }}
+      needs: Lint
+      strategy:
+        matrix:
+          os: [ubuntu-latest, windows-latest]
+      env:
+        HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
       steps:
           - name: Checkout repo
             uses: actions/checkout@v2
+
           - name: Install uv
             uses: astral-sh/setup-uv@v5
 
           - name: Set up Python
             uses: actions/setup-python@v2
             with:
                 python-version: '3.11'
-              
           - name: Install package
             run: uv pip install -e .[dev] --system
+
           - name: Download data inputs
             run: make download
             env:
               HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
-              POLICYENGINE_US_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_US_DATA_GITHUB_TOKEN }}
+
           - name: Build datasets
             run: make data
             env:
               TEST_LITE: true
               PYTHON_LOG_LEVEL: INFO  
           - name: Run tests
             run: pytest
+
           - name: Test documentation builds
-            run: make documentation
+            run: make documentation

Makefile

@@ -35,6 +35,7 @@ documentation:
 
 
 data:
+	python policyengine_us_data/utils/uprating.py
 	python policyengine_us_data/datasets/acs/acs.py
 	python policyengine_us_data/datasets/cps/cps.py
 	python policyengine_us_data/datasets/puf/irs_puf.py
@@ -43,8 +44,9 @@ data:
 	python policyengine_us_data/datasets/cps/enhanced_cps.py
 
 clean:
-	rm -f policyengine_us_data/storage/puf_2015.csv
-	rm -f policyengine_us_data/storage/demographics_2015.csv
+	rm -f policyengine_us_data/storage/*.h5
+	git clean -fX -- '*.csv'
+	rm -rf policyengine_us_data/docs/_build
 
 build:
 	python -m build

README.md

@@ -2,9 +2,17 @@
 
 ## Installation
 
+While it is possible to install via PyPi:
 ```bash
 pip install policyengine-us-data
 ```
+the recommended installion is 
+```
+pip install -e .[dev]
+```
+which installs the development dependencies in a reference-only manner (so that changes
+to the package code will be reflected immediately); `policyengine-us-data` is a dev package
+and not intended for direct access.
 
 ## Building the Paper
 

docs/add_plotly_to_book.py

@@ -14,7 +14,7 @@
 book_folder = Path(args.book_path)
 
 for html_file in book_folder.glob("**/*.html"):
-    with open(html_file, "r") as f:
+    with open(html_file, "r", encoding="utf-8") as f:
         html = f.read()
 
     # Add the script tag to the start of the <head> tag.
@@ -23,5 +23,5 @@
         '<head><script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>',
     )
 
-    with open(html_file, "w") as f:
+    with open(html_file, "w", encoding="utf-8") as f:
         f.write(html)

policyengine_us_data/datasets/cps/cps.py

@@ -1,3 +1,4 @@
+from importlib.resources import files
 from policyengine_core.data import Dataset
 from policyengine_us_data.storage import STORAGE_FOLDER
 import h5py
@@ -529,10 +530,13 @@ def add_personal_income_variables(
         year (int): The CPS year
     """
     # Get income imputation parameters.
-    yamlfilename = os.path.join(
-        os.path.abspath(os.path.dirname(__file__)),
-        "imputation_parameters.yaml",
+    yamlfilename = (
+        files("policyengine_us_data")
+        / "datasets"
+        / "cps"
+        / "imputation_parameters.yaml"
     )
+
     with open(yamlfilename, "r", encoding="utf-8") as yamlfile:
         p = yaml.safe_load(yamlfile)
     assert isinstance(p, dict)

policyengine_us_data/datasets/puf/uprate_puf.py

@@ -2,6 +2,7 @@
 import numpy as np
 from policyengine_us_data.storage import STORAGE_FOLDER
 
+
 ITMDED_GROW_RATE = 0.02  # annual growth rate in itemized deduction amounts
 
 USE_VARIABLE_SPECIFIC_POPULATION_GROWTH_DIVISORS = False
@@ -95,7 +96,6 @@ def get_soi_aggregate(variable, year, is_count):
     if variable == "adjusted_gross_income" and is_count:
         # AGI isn't treated like the other variables
         return get_soi_aggregate("count", year, True)
-
     is_variable = soi.Variable == variable
     is_year = soi.Year == year
     filing_status = soi["Filing status"] == "All"