From 196388e906f588265c100f8f09d5c166f86bf524 Mon Sep 17 00:00:00 2001 From: Tieqiong Zhang Date: Fri, 1 Aug 2025 00:01:21 -0400 Subject: [PATCH 1/2] block fast fail linux wheels for runtime libraries --- .github/workflows/_build-non-pure-package.yml | 71 ++++++++++++++++--- .github/workflows/_build-python-package.yml | 26 +++++++ .../workflows/_build-wheel-release-upload.yml | 33 ++++----- 3 files changed, 100 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/_build-python-package.yml diff --git a/.github/workflows/_build-non-pure-package.yml b/.github/workflows/_build-non-pure-package.yml index 4ca0b5c..785ef32 100644 --- a/.github/workflows/_build-non-pure-package.yml +++ b/.github/workflows/_build-non-pure-package.yml @@ -2,6 +2,12 @@ name: Build wheel and sdist for a non-pure Python package on: workflow_call: + inputs: + project: + description: 'Name of the project to build' + default: 'PROJECT_NAME' + required: true + type: string jobs: build_sdist: @@ -79,21 +85,66 @@ jobs: conda config --set always_yes yes --set changeps1 no - - name: Install requirements + - name: Install requirements and build wheels run: | conda install --file requirements/conda.txt conda install --file requirements/pip.txt pip install --upgrade build - - - - name: Build and repair wheels - run: | python -m pip wheel --no-deps --wheel-dir ./dist . - if [[ "${{ matrix.buildplat }}" == "ubuntu-latest" ]]; then - conda install auditwheel patchelf - LD_LIBRARY_PATH=$CONDA_PREFIX/lib auditwheel repair dist/*.whl -w dist/ - rm dist/*-linux_*.whl - fi + + - name: Repair wheels + if: matrix.buildplat == 'ubuntu-latest' + run: | + conda install auditwheel patchelf + LD_LIBRARY_PATH=$CONDA_PREFIX/lib auditwheel repair dist/*.whl -w dist/ + + set -euo pipefail + project="${{ inputs.project }}" + project_path="${project//./\/}" + project_pkg="${project//./_}" + + dist_dir="$(pwd)/dist" + for whl in "$dist_dir"/*manylinux*.whl; do + echo "Patching $whl" + tmp=$(mktemp -d) + unzip -q "$whl" -d "$tmp" + + libsdir="$tmp/${project_pkg}.libs" + declare -A vend_map + + for f in "$libsdir"/*.so*; do + base=$(basename "$f") + if [[ $base =~ ^(.+)-([0-9a-f]{8})(\.so.*)$ ]]; then + real="${BASH_REMATCH[1]}${BASH_REMATCH[3]}" + else + real="$base" + fi + vend_map["$base"]="$real" + done + + find "$tmp/$project_path" -type f -name "*.so" | while read -r so; do + for vend in "${!vend_map[@]}"; do + real=${vend_map[$vend]} + echo " replace-needed $vend → $real in $so" + patchelf --replace-needed "$vend" "$real" "$so" + done + if [[ "$project" == *.* ]]; then + rpath='$ORIGIN/../../../../' + else + rpath='$ORIGIN/../../../' + fi + echo " set-rpath in $so" + patchelf --set-rpath "$rpath" "$so" + done + + rm -f "$libsdir"/*.so* + rm -f "$whl" + ( cd "$tmp" && zip -q -FS -r "$whl" . ) + rm -rf "$tmp" + done + + echo "Patch done!" + rm dist/*-linux_*.whl - name: Upload wheels to GitHub uses: actions/upload-artifact@v4 diff --git a/.github/workflows/_build-python-package.yml b/.github/workflows/_build-python-package.yml new file mode 100644 index 0000000..60509a4 --- /dev/null +++ b/.github/workflows/_build-python-package.yml @@ -0,0 +1,26 @@ +name: Build Wheels + +on: + workflow_call: + inputs: + project: + description: 'Name of the project to build' + default: 'PROJECT_NAME' + required: true + type: string + c_extension: + description: 'Whether the project has a C extension' + default: false + required: true + type: boolean + +jobs: + pure-python: + if: inputs.c_extension == false + uses: ./.github/workflows/_build-pure-python-package.yml + + non-pure-python: + if: inputs.c_extension + uses: ./.github/workflows/_build-non-pure-package.yml + with: + project: ${{ inputs.project }} diff --git a/.github/workflows/_build-wheel-release-upload.yml b/.github/workflows/_build-wheel-release-upload.yml index ddc541d..9f0f0f9 100644 --- a/.github/workflows/_build-wheel-release-upload.yml +++ b/.github/workflows/_build-wheel-release-upload.yml @@ -6,12 +6,12 @@ on: project: description: 'Name of the project to build and release' default: 'PROJECT_NAME' - required: false + required: true type: string c_extension: description: 'Whether the project has a C extension' default: false - required: false + required: true type: boolean maintainer_github_username: description: GitHub username authorized to start GitHub/PyPI release' @@ -31,21 +31,16 @@ jobs: with: maintainer_github_username: ${{ inputs.maintainer_github_username }} - build-pure-python-package: - needs: [tag-privilege-check] - if: inputs.c_extension == false - uses: ./.github/workflows/_build-pure-python-package.yml - - build-non-pure-python-package: + build-python-package: needs: [tag-privilege-check] - if: inputs.c_extension - uses: ./.github/workflows/_build-non-pure-package.yml + uses: ./.github/workflows/_build-python-package.yml + with: + project: ${{ inputs.project }} + c_extension: ${{ inputs.c_extension }} update-changelog: - # The always() function is necessary to ensure that we wait for both build jobs to complete, even if one of them is skipped. - # Without always(), if one of the needed jobs is skipped, the `update-changelog` job will not be executed. - if: "always() && !contains(github.ref, 'rc')" - needs: [build-pure-python-package, build-non-pure-python-package] + if: "!contains(github.ref, 'rc')" + needs: [build-python-package] uses: ./.github/workflows/_update_changelog.yml secrets: PAT_TOKEN: ${{ secrets.PAT_TOKEN }} @@ -56,19 +51,19 @@ jobs: # tag to reflect the latest changes in the CHANGELOG.rst done by the update-changelog above. # For more discussions, please read https://github.com/scikit-package/release-scripts/pull/120 # Always run this job for a full release but fail if the update-changelog job previously failed. - if: "always() && !contains(github.ref, 'rc')" + if: "!contains(github.ref, 'rc')" needs: [update-changelog] uses: ./.github/workflows/_delete_create_tag.yml with: update_changelog_result: ${{ needs.update-changelog.result }} github-pre-release: - if: "always() && contains(github.ref, 'rc')" - needs: [build-pure-python-package, build-non-pure-python-package] + if: "contains(github.ref, 'rc')" + needs: [build-python-package] uses: ./.github/workflows/_github_pre_release.yml github-release: - if: "always() && !contains(github.ref, 'rc')" + if: "!contains(github.ref, 'rc')" needs: [delete-create-new-tag] uses: ./.github/workflows/_github_release.yml with: @@ -77,7 +72,6 @@ jobs: pypi-publish: needs: [github-pre-release, github-release] runs-on: ubuntu-latest - if: always() # This job will always initiate regardless of the success or failure of the needed jobs steps: - name: Fail pypi-publish job if github-(pre)-release job failed run: | @@ -113,7 +107,6 @@ jobs: docs: needs: [github-pre-release, github-release] - if: always() uses: ./.github/workflows/_publish-docs-on-release.yml with: project: ${{ inputs.project }} From 6c41320ee9fb2b10a78f59c900edf8503d39fe31 Mon Sep 17 00:00:00 2001 From: Tieqiong Zhang Date: Fri, 1 Aug 2025 00:41:23 -0400 Subject: [PATCH 2/2] pcmt --- .github/workflows/_build-non-pure-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_build-non-pure-package.yml b/.github/workflows/_build-non-pure-package.yml index 785ef32..338b08e 100644 --- a/.github/workflows/_build-non-pure-package.yml +++ b/.github/workflows/_build-non-pure-package.yml @@ -91,7 +91,7 @@ jobs: conda install --file requirements/pip.txt pip install --upgrade build python -m pip wheel --no-deps --wheel-dir ./dist . - + - name: Repair wheels if: matrix.buildplat == 'ubuntu-latest' run: |