diff --git a/.github/workflows/code_changes.yaml b/.github/workflows/code_changes.yaml new file mode 100644 index 00000000..6115c945 --- /dev/null +++ b/.github/workflows/code_changes.yaml @@ -0,0 +1,47 @@ +# Workflow that runs on code changes to a pull request. + +name: Code changes +on: + push: + branches: + - main + + paths: + - policyengine_us_data/** + - tests/** + +jobs: + Lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check formatting + uses: "lgeiger/black-action@master" + with: + args: ". -l 79 --check" + Test: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v2 + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.10' + + - name: Install package + run: uv pip install -e .[dev] --system + - name: Download data inputs + run: make download + env: + HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + POLICYENGINE_US_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_US_DATA_GITHUB_TOKEN }} + - name: Build datasets + run: make data + - name: Run tests + run: pytest + - name: Test documentation builds + run: make documentation \ No newline at end of file diff --git a/.github/workflows/pr_code_changes.yaml b/.github/workflows/pr_code_changes.yaml new file mode 100644 index 00000000..62cbfb7d --- /dev/null +++ b/.github/workflows/pr_code_changes.yaml @@ -0,0 +1,49 @@ +# Workflow that runs on code changes to a pull request. + +name: Code changes +on: + pull_request: + branches: + - main + + paths: + - policyengine_us_data/** + - tests/** + +jobs: + Lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check formatting + uses: "lgeiger/black-action@master" + with: + args: ". -l 79 --check" + Test: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v2 + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.10' + + - name: Install package + run: uv pip install -e .[dev] --system + - name: Download data inputs + run: make download + env: + HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + POLICYENGINE_US_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_US_DATA_GITHUB_TOKEN }} + - name: Build datasets + run: make data + env: + LITE_MODE: true + - name: Run tests + run: pytest + - name: Test documentation builds + run: make documentation \ No newline at end of file diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 00000000..661a042f --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,29 @@ +name: Push +on: + push: + branches: [ main ] + paths: + - pyproject.toml + +jobs: + Publish: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + - name: Publish a git tag + run: ".github/publish-git-tag.sh || true" + - name: Install package + run: make install + - name: Build package + run: make + - name: Publish a Python distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI }} + skip-existing: true \ No newline at end of file diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml deleted file mode 100644 index f373c00c..00000000 --- a/.github/workflows/pull_request.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# This workflow is used to check various parts of the -# run prior to merging: - -# - The code's formatting -# - Versioning - note that this checks, but does not update -# - That the code builds successfully -# - That any automated tests pass -name: Pull request - -on: - pull_request: - branches: [main] - -jobs: - lint: - runs-on: ubuntu-latest - name: Lint - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install black - - name: Check formatting - run: black . -l 79 --check - check-version: - name: Check version - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch all history for all tags and branches - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Build changelog - run: pip install "yaml-changelog>=0.1.7" && make changelog - - name: Preview changelog update - run: ".github/get-changelog-diff.sh" - - name: Check version number has been properly updated - run: ".github/is-version-number-acceptable.sh" - test: - name: Build and test - runs-on: larger-runner - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch all history for all tags and branches - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Install package - run: make install - - name: Download data inputs - run: make download - env: - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} - POLICYENGINE_US_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_US_DATA_GITHUB_TOKEN }} - - name: Build datasets - run: make data - env: - LITE_MODE: true - - name: Run tests - run: pytest - - name: Test documentation builds - run: make documentation diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml deleted file mode 100644 index 846cdf66..00000000 --- a/.github/workflows/push.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# After successful versioning, this script runs various -# parts of the push process -name: Push - -on: - workflow_run: - workflows: ["Update versioning"] - types: [completed] - -jobs: - lint: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - name: Lint - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install black - - name: Check formatting - run: black . -l 79 --check - test: - name: Build and test - runs-on: larger-runner - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch all history for all tags and branches - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Install package - run: make install - - name: Download data inputs - run: make download - env: - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} - POLICYENGINE_US_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_US_DATA_GITHUB_TOKEN }} - - name: Build datasets - run: make data - - name: Run tests - run: pytest - - name: Upload data - run: make upload - env: - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} - - name: Test documentation builds - run: make documentation - - name: Build Jupyter Book - run: make documentation - - name: Deploy documentation - uses: JamesIves/github-pages-deploy-action@releases/v4 - with: - branch: gh-pages - folder: docs/_build/html - publish-to-pypi: - name: Publish to PyPI - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch all history for all tags and branches - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Install package - run: make install - - name: Build package - run: python -m build - - name: Publish a git tag - run: ".github/publish-git-tag.sh || true" - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI }} - skip-existing: true - docker: - name: Docker - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - name: Checkout repo - uses: actions/checkout@v4 - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{github.actor}} - password: ${{secrets.GITHUB_TOKEN}} - - name: Build container - run: docker build . -f docker/policyengine_us_data.Dockerfile -t ghcr.io/policyengine/policyengine-us-data:latest - - name: Push container - run: docker push ghcr.io/policyengine/policyengine-us-data:latest \ No newline at end of file diff --git a/.github/workflows/update_versioning.yaml b/.github/workflows/update_versioning.yaml deleted file mode 100644 index d5fd5094..00000000 --- a/.github/workflows/update_versioning.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# This builds and pushes the changelog, then -# automatically runs push_2 using the new version number -# to allow for proper pushing to PyPI. - -# This script must run first and complete to allow for -# proper versioning. - -name: Update versioning - -on: - push: - branches: [main] - -jobs: - check-version: - name: Check version - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch all history for all tags and branches - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Build changelog - run: pip install "yaml-changelog>=0.1.7" && make changelog - - name: Preview changelog update - run: ".github/get-changelog-diff.sh" - - name: Check version number has been properly updated - run: ".github/is-version-number-acceptable.sh" - - name: Update changelog - uses: EndBug/add-and-commit@v9 - with: - add: "." - committer_name: Github Actions[bot] - author_name: Github Actions[bot] - message: Update PolicyEngine US data - github_token: ${{ secrets.POLICYENGINE_US_DATA_GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/versioning.yaml b/.github/workflows/versioning.yaml new file mode 100644 index 00000000..b5185793 --- /dev/null +++ b/.github/workflows/versioning.yaml @@ -0,0 +1,34 @@ +# Workflow that runs on versioning metadata updates. + +name: Versioning updates +on: + push: + branches: + - main + + paths: + - changelog_entry.yaml + +jobs: + Versioning: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + token: ${{ secrets.POLICYENGINE_GITHUB }} + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + - name: Build changelog + run: pip install yaml-changelog && make changelog + - name: Preview changelog update + run: ".github/get-changelog-diff.sh" + - name: Update changelog + uses: EndBug/add-and-commit@v9 + with: + add: "." + message: Update package version \ No newline at end of file diff --git a/policyengine_us_data/datasets/cps/extended_cps.py b/policyengine_us_data/datasets/cps/extended_cps.py index b0ce439d..dbccf0c1 100644 --- a/policyengine_us_data/datasets/cps/extended_cps.py +++ b/policyengine_us_data/datasets/cps/extended_cps.py @@ -149,12 +149,6 @@ def generate(self): self.time_period: values, } - # Assume interest expenses is all deductible mortgage interest. - - new_data["deductible_mortgage_interest"] = { - self.time_period: new_data["interest_deduction"][self.time_period] - } - self.save_dataset(new_data) diff --git a/policyengine_us_data/datasets/puf/puf.py b/policyengine_us_data/datasets/puf/puf.py index 34b52cbe..53d35748 100644 --- a/policyengine_us_data/datasets/puf/puf.py +++ b/policyengine_us_data/datasets/puf/puf.py @@ -281,6 +281,7 @@ def preprocess_puf(puf: pd.DataFrame) -> pd.DataFrame: "unreported_payroll_tax", "pre_tax_contributions", "w2_wages_from_qualified_business", + "deductible_mortgage_interest", ] @@ -431,7 +432,16 @@ def add_filer(self, row, tax_unit_id): self.holder["household_weight"].append(row["household_weight"]) self.holder["is_male"].append(row["GENDER"] == 1) + # Assume all of the interest deduction is the filer's deductible mortgage interest + + self.holder["deductible_mortgage_interest"].append( + row["interest_deduction"] + ) + for key in FINANCIAL_SUBSET: + if key == "deductible_mortgage_interest": + # Skip this one- we are adding it artificially at the filer level. + continue if self.variable_to_entity[key] == "person": self.holder[key].append(row[key] * self.earn_splits[-1]) @@ -457,7 +467,14 @@ def add_spouse(self, row, tax_unit_id): opposite_gender_code if is_opposite_gender else same_gender_code ) + # Assume all of the interest deduction is the filer's deductible mortgage interest + + self.holder["deductible_mortgage_interest"].append(0) + for key in FINANCIAL_SUBSET: + if key == "deductible_mortgage_interest": + # Skip this one- we are adding it artificially at the filer level. + continue if self.variable_to_entity[key] == "person": self.holder[key].append(row[key] * (1 - self.earn_splits[-1])) @@ -474,7 +491,14 @@ def add_dependent(self, row, tax_unit_id, dependent_id): age = decode_age_dependent(round(row[f"AGEDP{dependent_id + 1}"])) self.holder["age"].append(age) + # Assume all of the interest deduction is the filer's deductible mortgage interest + + self.holder["deductible_mortgage_interest"].append(0) + for key in FINANCIAL_SUBSET: + if key == "deductible_mortgage_interest": + # Skip this one- we are adding it artificially at the filer level. + continue if self.variable_to_entity[key] == "person": self.holder[key].append(0)