diff --git a/.github/workflows/build-wheels-defined.yml b/.github/workflows/build-wheels-defined.yml index db47e42..0c43309 100644 --- a/.github/workflows/build-wheels-defined.yml +++ b/.github/workflows/build-wheels-defined.yml @@ -157,6 +157,8 @@ jobs: run: os_dependencies/macos.sh - name: Build wheels + env: + ARCHFLAGS: "-arch x86_64" # Force x86_64-only wheels, not universal2 run: | python build_wheels_from_file.py --requirements ${{ inputs.packages }} @@ -207,6 +209,8 @@ jobs: run: os_dependencies/macos.sh - name: Build wheels + env: + ARCHFLAGS: "-arch arm64" # Force arm64-only wheels, not universal2 run: | python build_wheels_from_file.py --requirements ${{ inputs.packages }} @@ -221,33 +225,39 @@ jobs: needs: get-supported-versions name: linux aarch32 (armv7) if: ${{ inputs.os_linux_armv7 }} - runs-on: linux-armv7-self-hosted + runs-on: ubuntu-latest strategy: fail-fast: false matrix: python-version: ${{ fromJson(needs.get-supported-versions.outputs.supported_python) }} - container: python:${{ matrix.python-version }}-bookworm steps: + - name: Set up QEMU for ARMv7 + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/arm/v7 + - name: Checkout repository uses: actions/checkout@v4 - - name: Get Python version - run: | - python --version - python -m pip install --upgrade pip - - - name: Install build dependencies - run: python -m pip install -r build_requirements.txt - - - name: Install additional OS dependencies - Linux ARM - run: os_dependencies/linux_arm.sh - - - name: Build wheels + - name: Build wheels - ARMv7 (in Docker) run: | - # Rust directory needs to be included for Linux ARM7 - . $HOME/.cargo/env - - python build_wheels_from_file.py --requirements ${{ inputs.packages }} + docker run --rm --platform linux/arm/v7 \ + -v $(pwd):/work \ + -w /work \ + -e GH_TOKEN="${GH_TOKEN}" \ + -e PIP_NO_CACHE_DIR=1 \ + python:${{ matrix.python-version }}-bookworm \ + bash -c " + set -e + python --version + # Install pip packages without cache to reduce memory usage + python -m pip install --no-cache-dir --upgrade pip + python -m pip install --no-cache-dir -r build_requirements.txt + bash os_dependencies/linux_arm.sh + # Source Rust environment after installation + . \$HOME/.cargo/env + python build_wheels_from_file.py --requirements '${{ inputs.packages }}' + " - name: Upload artifacts of downloaded_wheels directory uses: actions/upload-artifact@v4 @@ -291,9 +301,17 @@ jobs: name: wheels-download-directory-linux-arm64-${{ matrix.python-version }} path: ./downloaded_wheels - upload-python-wheels: + # Repair wheels for dynamically linked libraries on all platforms + # https://github.com/espressif/idf-python-wheels/blob/main/README.md#universal-wheel-tag---linking-of-dynamic-libraries + repair-wheels: if: ${{ always() }} needs: [get-supported-versions, ubuntu-latest, windows-latest, macos-latest, macos-m1, linux-armv7, linux-arm64] + name: Repair wheels + uses: ./.github/workflows/wheels-repair.yml + + upload-python-wheels: + if: ${{ always() }} + needs: [repair-wheels] name: Upload Python wheels uses: espressif/idf-python-wheels/.github/workflows/upload-python-wheels.yml@main secrets: inherit diff --git a/.github/workflows/build-wheels-platforms.yml b/.github/workflows/build-wheels-platforms.yml index 0b33ca7..ce97ba8 100644 --- a/.github/workflows/build-wheels-platforms.yml +++ b/.github/workflows/build-wheels-platforms.yml @@ -17,33 +17,52 @@ jobs: build-wheels: needs: get-supported-versions name: Build for ${{ matrix.os }} (Python ${{matrix.python-version}}) - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: - os: # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories - - windows-latest - - ubuntu-latest - - macos-15-intel # MacOS x86_64 - - macos-latest # MacOS arm64 (M1) - - ubuntu-24.04-arm - - linux-armv7-self-hosted + os: # Platform names for identification + - Windows + - Linux x86_64 + - macOS Intel + - macOS ARM + - Linux ARM64 + - Linux ARMv7 include: - - os: linux-armv7-self-hosted - CONTAINER: python:${{ needs.get-supported-versions.outputs.oldest_supported_python }}-bookworm + - os: Windows + runner: windows-latest + arch: windows-x86_64 + - os: Linux x86_64 + runner: ubuntu-latest + arch: linux-x86_64 + - os: macOS Intel + runner: macos-15-intel + arch: macos-x86_64 + - os: macOS ARM + runner: macos-latest + arch: macos-arm64 + - os: Linux ARM64 + runner: ubuntu-24.04-arm + arch: linux-arm64 + - os: Linux ARMv7 + runner: ubuntu-latest + arch: linux-armv7 python-version: ['${{ needs.get-supported-versions.outputs.oldest_supported_python }}'] - # Use python container on ARM - container: ${{ matrix.CONTAINER }} - steps: + - name: Set up QEMU for ARMv7 + if: matrix.os == 'Linux ARMv7' + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/arm/v7 + - name: OS info - if: matrix.os != 'windows-latest' + if: matrix.os != 'Windows' run: | echo "Operating System: ${{ runner.os }}" echo "Architecture: $(uname -m)" - name: OS info - if: matrix.os == 'windows-latest' + if: matrix.os == 'Windows' run: | echo "Operating System: ${{ runner.os }}" echo "Architecture: $env:PROCESSOR_ARCHITECTURE" @@ -60,66 +79,92 @@ jobs: - name: Setup Python # Skip setting python on ARM because of missing compatibility: https://github.com/actions/setup-python/issues/108 - if: matrix.os != 'linux-armv7-self-hosted' + if: matrix.os != 'Linux ARMv7' uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install build dependencies + if: matrix.os != 'Linux ARMv7' run: | python -m pip install --upgrade pip python -m pip install -r build_requirements.txt - - name: Get Tools versions + if: matrix.os != 'Linux ARMv7' run: | python --version pip show pip setuptools - - name: Install additional OS dependencies - Ubuntu - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'Linux x86_64' run: os_dependencies/ubuntu.sh - name: Install additional OS dependencies - MacOS - if: matrix.os == 'macos-latest' || matrix.os == 'macos-15-intel' + if: matrix.os == 'macOS ARM' || matrix.os == 'macOS Intel' run: os_dependencies/macos.sh - - name: Install additional OS dependencies - Linux ARM - if: matrix.os == 'linux-armv7-self-hosted' || matrix.os == 'ubuntu-24.04-arm' + - name: Install additional OS dependencies - Linux ARM64 + if: matrix.os == 'Linux ARM64' run: os_dependencies/linux_arm.sh - name: Install additional OS dependencies - Windows - if: matrix.os == 'windows-latest' + if: matrix.os == 'Windows' run: powershell -ExecutionPolicy Bypass -File os_dependencies/windows.ps1 - - - name: Build wheels for IDF - if: matrix.os != 'windows-latest' + - name: Build wheels for IDF - ARMv7 (in Docker) + if: matrix.os == 'Linux ARMv7' run: | - # Source Rust environment for self-hosted ARMv7 runner - if [ "${{ matrix.os }}" = "linux-armv7-self-hosted" ]; then - . $HOME/.cargo/env + docker run --rm --platform linux/arm/v7 \ + -v $(pwd):/work \ + -w /work \ + -e MIN_IDF_MAJOR_VERSION=${{ needs.get-supported-versions.outputs.min_idf_major_version }} \ + -e MIN_IDF_MINOR_VERSION=${{ needs.get-supported-versions.outputs.min_idf_minor_version }} \ + -e GH_TOKEN="${GH_TOKEN}" \ + -e PIP_NO_CACHE_DIR=1 \ + python:${{ matrix.python-version }}-bookworm \ + bash -c " + set -e + python --version + # Install pip packages without cache to reduce memory usage + python -m pip install --no-cache-dir --upgrade pip + python -m pip install --no-cache-dir -r build_requirements.txt + bash os_dependencies/linux_arm.sh + # Source Rust environment after installation + . \$HOME/.cargo/env + python build_wheels.py + " + + - name: Build wheels for IDF - Linux/macOS + if: matrix.os != 'Windows' && matrix.os != 'Linux ARMv7' + run: | + # Set ARCHFLAGS for macOS to prevent universal2 wheels + if [ "${{ matrix.os }}" = "macOS ARM" ]; then + export ARCHFLAGS="-arch arm64" + elif [ "${{ matrix.os }}" = "macOS Intel" ]; then + export ARCHFLAGS="-arch x86_64" fi python build_wheels.py - name: Build wheels for IDF - Windows - if: matrix.os == 'windows-latest' + if: matrix.os == 'Windows' run: python build_wheels.py - name: Upload artifacts of downloaded_wheels directory uses: actions/upload-artifact@v4 with: - name: wheels-download-directory-${{ matrix.os}}-${{ matrix.python-version }} + name: wheels-download-directory-${{ matrix.arch }}-${{ matrix.python-version }} path: ./downloaded_wheels + retention-days: 1 - name: Upload artifacts of Python version dependent wheels uses: actions/upload-artifact@v4 with: - name: dependent_requirements_${{ matrix.os}} + name: dependent_requirements_${{ matrix.arch }} path: ./dependent_requirements.txt + retention-days: 1 build-python-version-dependent-wheels: @@ -130,9 +175,15 @@ jobs: supported_python_versions: ${{ needs.get-supported-versions.outputs.supported_python }} oldest_supported_python: ${{ needs.get-supported-versions.outputs.oldest_supported_python }} - # TODO Uncomment this when we are ready to upload the wheels - #upload-python-wheels: - # needs: [build-wheels, build-python-version-dependent-wheels] - # name: Upload Python wheels - # uses: ./.github/workflows/upload-python-wheels.yml - # secrets: inherit + # Repair wheels for dynamically linked libraries on all platforms + # https://github.com/espressif/idf-python-wheels/blob/main/README.md#universal-wheel-tag---linking-of-dynamic-libraries + repair-wheels: + needs: [build-wheels, build-python-version-dependent-wheels] + name: Repair wheels + uses: ./.github/workflows/wheels-repair.yml + + upload-python-wheels: + needs: [repair-wheels] + name: Upload Python wheels + uses: ./.github/workflows/upload-python-wheels.yml + secrets: inherit diff --git a/.github/workflows/build-wheels-python-dependent.yml b/.github/workflows/build-wheels-python-dependent.yml index 5b5f3a3..a207680 100644 --- a/.github/workflows/build-wheels-python-dependent.yml +++ b/.github/workflows/build-wheels-python-dependent.yml @@ -15,104 +15,150 @@ on: jobs: triage: name: ${{ matrix.os }} - ${{ matrix.python-version }} - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.runner }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} strategy: fail-fast: false matrix: os: - - windows-latest - - ubuntu-latest - - macos-15-intel # MacOS x86_64 - - macos-latest # MacOS arm64 (M1) - - ubuntu-24.04-arm - - linux-armv7-self-hosted + - Windows + - Linux x86_64 + - macOS Intel + - macOS ARM + - Linux ARM64 + - Linux ARMv7 + include: + - os: Windows + runner: windows-latest + arch: windows-x86_64 + - os: Linux x86_64 + runner: ubuntu-latest + arch: linux-x86_64 + - os: macOS Intel + runner: macos-15-intel + arch: macos-x86_64 + - os: macOS ARM + runner: macos-latest + arch: macos-arm64 + - os: Linux ARM64 + runner: ubuntu-24.04-arm + arch: linux-arm64 + - os: Linux ARMv7 + runner: ubuntu-latest + arch: linux-armv7 python-version: ${{ fromJson(inputs.supported_python_versions) }} exclude: # Exclude oldest supported Python since it's already built in the platform builds - python-version: ${{ inputs.oldest_supported_python }} - os: windows-latest + os: Windows - python-version: ${{ inputs.oldest_supported_python }} - os: ubuntu-latest + os: Linux x86_64 - python-version: ${{ inputs.oldest_supported_python }} - os: macos-15-intel + os: macOS Intel - python-version: ${{ inputs.oldest_supported_python }} - os: macos-latest + os: macOS ARM - python-version: ${{ inputs.oldest_supported_python }} - os: ubuntu-24.04-arm + os: Linux ARM64 - python-version: ${{ inputs.oldest_supported_python }} - os: linux-armv7-self-hosted - - # Use python container on ARM - dynamically constructed with bookworm (Debian 12) - container: ${{matrix.os == 'linux-armv7-self-hosted' && format('python:{0}-bookworm', matrix.python-version) || null}} + os: Linux ARMv7 steps: + - name: Set up QEMU for ARMv7 + if: matrix.os == 'Linux ARMv7' + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/arm/v7 + - name: Checkout repository uses: actions/checkout@v4 - name: Setup Python - # Skip setting python on ARM because of missing compatibility: https://github.com/actions/setup-python/issues/108 - if: matrix.os != 'linux-armv7-self-hosted' + # Skip setting python on ARMv7 (runs in Docker) + if: matrix.os != 'Linux ARMv7' uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Get Python version + if: matrix.os != 'Linux ARMv7' run: | python --version python -m pip install --upgrade pip - - name: Install dependencies + if: matrix.os != 'Linux ARMv7' run: python -m pip install -r build_requirements.txt - name: Install additional OS dependencies - Ubuntu - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'Linux x86_64' run: os_dependencies/ubuntu.sh - name: Install additional OS dependencies - MacOS - if: matrix.os == 'macos-latest' || matrix.os == 'macos-15-intel' + if: matrix.os == 'macOS ARM' || matrix.os == 'macOS Intel' run: os_dependencies/macos.sh - - - name: Install additional OS dependencies - Linux ARM7 - if: matrix.os == 'linux-armv7-self-hosted' || matrix.os == 'ubuntu-24.04-arm' + - name: Install additional OS dependencies - Linux ARM64 + if: matrix.os == 'Linux ARM64' run: os_dependencies/linux_arm.sh - name: Download artifacts uses: actions/download-artifact@v4 with: - name: dependent_requirements_${{ matrix.os}} - path: dependent_requirements_${{ matrix.os}} + name: dependent_requirements_${{ matrix.arch }} + path: dependent_requirements_${{ matrix.arch }} - name: Print requirements - if: matrix.os != 'windows-latest' - run: cat dependent_requirements_${{ matrix.os}}/dependent_requirements.txt + if: matrix.os != 'Windows' + run: cat dependent_requirements_${{ matrix.arch }}/dependent_requirements.txt - name: Print requirements - Windows - if: matrix.os == 'windows-latest' - run: type dependent_requirements_${{ matrix.os}}\\dependent_requirements.txt + if: matrix.os == 'Windows' + run: type dependent_requirements_${{ matrix.arch }}\\dependent_requirements.txt - - name: Build Python dependent wheels for ${{ matrix.python-version }} - if: matrix.os != 'windows-latest' + - name: Build Python dependent wheels - ARMv7 (in Docker) + if: matrix.os == 'Linux ARMv7' + run: | + docker run --rm --platform linux/arm/v7 \ + -v $(pwd):/work \ + -w /work \ + -e PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 \ + -e GH_TOKEN="${GH_TOKEN}" \ + -e PIP_NO_CACHE_DIR=1 \ + python:${{ matrix.python-version }}-bookworm \ + bash -c " + set -e + python --version + # Install pip packages without cache to reduce memory usage + python -m pip install --no-cache-dir --upgrade pip + python -m pip install --no-cache-dir -r build_requirements.txt + bash os_dependencies/linux_arm.sh + # Source Rust environment after installation + . \$HOME/.cargo/env + python build_wheels_from_file.py dependent_requirements_${{ matrix.arch }} + " + + - name: Build Python dependent wheels - Linux/macOS + if: matrix.os != 'Windows' && matrix.os != 'Linux ARMv7' run: | - # Source Rust environment for self-hosted ARMv7 runner - if [ "${{ matrix.os }}" = "linux-armv7-self-hosted" ]; then - export PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 - . $HOME/.cargo/env + # Set ARCHFLAGS for macOS to prevent universal2 wheels + if [ "${{ matrix.os }}" = "macOS ARM" ]; then + export ARCHFLAGS="-arch arm64" + elif [ "${{ matrix.os }}" = "macOS Intel" ]; then + export ARCHFLAGS="-arch x86_64" fi - python build_wheels_from_file.py dependent_requirements_${{ matrix.os}} + python build_wheels_from_file.py dependent_requirements_${{ matrix.arch }} - name: Build Python dependent wheels for ${{ matrix.python-version }} - Windows - if: matrix.os == 'windows-latest' - run: python build_wheels_from_file.py dependent_requirements_${{ matrix.os}} + if: matrix.os == 'Windows' + run: python build_wheels_from_file.py dependent_requirements_${{ matrix.arch }} - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: wheels-download-directory-${{ matrix.os }}-${{ matrix.python-version }} + name: wheels-download-directory-${{ matrix.arch }}-${{ matrix.python-version }} if-no-files-found: ignore path: ./downloaded_wheels retention-days: 1 diff --git a/.github/workflows/wheels-repair.yml b/.github/workflows/wheels-repair.yml new file mode 100644 index 0000000..0853022 --- /dev/null +++ b/.github/workflows/wheels-repair.yml @@ -0,0 +1,154 @@ +name: repair-wheels + +# Repairs wheels for dynamically linked libraries on all platforms +# https://github.com/espressif/idf-python-wheels/blob/main/README.md#universal-wheel-tag---linking-of-dynamic-libraries +# - Windows: delvewheel (bundles DLLs) +# - macOS: delocate (bundles dylibs) +# - Linux: auditwheel (bundles SOs) + +on: + workflow_call: + +jobs: + repair-wheels: + name: Repair ${{ matrix.platform }} wheels + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + platform: + - Windows + - macOS ARM + - macOS Intel + - Linux x86_64 + - Linux ARM64 + - Linux ARMv7 + include: + - platform: Windows + runner: windows-latest + tool: delvewheel + arch: windows-x86_64 + - platform: macOS ARM + runner: macos-latest + tool: delocate + arch: macos-arm64 + - platform: macOS Intel + runner: macos-15-intel + tool: delocate + arch: macos-x86_64 + - platform: Linux x86_64 + runner: ubuntu-latest + tool: auditwheel + manylinux_platform: manylinux_2_34_x86_64 + arch: linux-x86_64 + - platform: Linux ARM64 + runner: ubuntu-latest + tool: auditwheel + manylinux_platform: manylinux_2_34_aarch64 + setup_qemu: true + qemu_platform: arm64 + docker_platform: linux/arm64 + arch: linux-arm64 + - platform: Linux ARMv7 + runner: ubuntu-latest + tool: auditwheel + manylinux_platform: idf-python-wheels-armv7l + docker_image: ghcr.io/espressif/github-esp-dockerfiles/idf-python-wheels-armv7l:v1 + setup_qemu: true + qemu_platform: arm + docker_platform: linux/arm/v7 + arch: linux-armv7 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download wheel artifacts + uses: actions/download-artifact@v4 + with: + pattern: wheels-download-directory-* + path: ./downloaded_wheels + merge-multiple: true + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Set up QEMU + if: matrix.setup_qemu + uses: docker/setup-qemu-action@v3 + with: + platforms: ${{ matrix.qemu_platform }} + + - name: Install repair tool - Windows + if: matrix.tool == 'delvewheel' + run: python -m pip install delvewheel + + - name: Install repair tool - macOS + if: matrix.tool == 'delocate' + run: python -m pip install delocate + + - name: Install OS dependencies - macOS + if: matrix.tool == 'delocate' + run: bash os_dependencies/macos.sh + + - name: Install dependencies and repair - Windows/macOS + if: matrix.tool != 'auditwheel' + run: | + python -m pip install -r build_requirements.txt + python repair_wheels.py + + - name: Repair Linux x86_64 wheels in manylinux container + if: matrix.platform == 'Linux x86_64' + run: | + docker run --rm \ + -v $(pwd):/work \ + -w /work \ + quay.io/pypa/${{ matrix.manylinux_platform }} \ + bash -c " + bash os_dependencies/manylinux.sh + python3 -m ensurepip --default-pip || curl -sS https://bootstrap.pypa.io/get-pip.py | python3 + python3 -m pip install --upgrade pip + python3 -m pip install --ignore-installed -r build_requirements.txt + python3 repair_wheels.py + " + + - name: Repair Linux ARM64 wheels in manylinux container + if: matrix.platform == 'Linux ARM64' + run: | + docker run --rm \ + --platform ${{ matrix.docker_platform }} \ + -v $(pwd):/work \ + -w /work \ + quay.io/pypa/${{ matrix.manylinux_platform }} \ + bash -c " + bash os_dependencies/manylinux.sh + python3 -m ensurepip --default-pip || curl -sS https://bootstrap.pypa.io/get-pip.py | python3 + python3 -m pip install --upgrade pip + python3 -m pip install --ignore-installed -r build_requirements.txt + python3 repair_wheels.py + " + + - name: Repair Linux ARMv7 wheels in manylinux container + if: matrix.platform == 'Linux ARMv7' + run: | + docker run --rm \ + --platform ${{ matrix.docker_platform }} \ + -v $(pwd):/work \ + -w /work \ + ${{ matrix.docker_image }} \ + bash -c " + bash os_dependencies/linux_arm.sh + python3 -m pip install --break-system-packages --upgrade pip + python3 -m pip install --break-system-packages -r build_requirements.txt + python3 repair_wheels.py + " + + - name: Re-upload artifacts with repaired wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-repaired-${{ matrix.arch }} + path: ./downloaded_wheels + overwrite: true + retention-days: 1 diff --git a/README.md b/README.md index 2104704..d6eb67f 100644 --- a/README.md +++ b/README.md @@ -118,9 +118,20 @@ File for the requirements needed for the build process and the build script. ### os_dependencies When there is a need for additional OS dependencies to successfully build the wheels on a specific platform and architecture, the `.sh` script in the `os_dependencies` directory can be adjusted. +## Universal wheel tag - linking of dynamic libraries +The repair tools are used after build to link and bundle all the needed libraries into the wheel to produce correct universal tag and working wheel. If this is not able to achieve the broken wheel is deleted and not published to Espressif's PyPI. + +- [`auditwheel`](https://github.com/pypa/auditwheel) package to repair Linux's `manylinux` wheels +- [`delocate`](https://github.com/matthew-brett/delocate) package to repair Mac's dynamically linked libraries +- [`delvewheel`](https://github.com/adang1345/delvewheel) package to repair Windows's DLLs + +This logic is done by the [repair workflow](./.github/workflows/wheels-repair.yml) and the [`repair_wheels.py` script](./repair_wheels.py) + ## Activity Diagram The main file is `build-wheels-platforms.yml` which is scheduled to run periodically to build Python wheels for any requirement of all [ESP-IDF]-supported versions. -![IDF Python wheels - Activity diagram](./resources/idf-python-wheels_diagram.svg "IDF Python wheels - Activity diagram") + + +![IDF Python wheels - Activity diagram](./resources/idf-python-wheels-diagram.svg "IDF Python wheels - Activity diagram") *The diagram was generated with the open-source tool [PlantUML](https://plantuml.com) (and edited)* diff --git a/docker/Dockerfile.manylinux_armv7l b/docker/Dockerfile.manylinux_armv7l new file mode 100644 index 0000000..dcb8414 --- /dev/null +++ b/docker/Dockerfile.manylinux_armv7l @@ -0,0 +1,95 @@ +# Minimal manylinux-compatible Docker image for ARMv7l (32-bit ARM) +# Based on Debian Bookworm for broad compatibility + +FROM arm32v7/debian:bookworm + +LABEL maintainer="Espressif Systems" +LABEL description="Minimal manylinux-compatible environment for building Python wheels on ARMv7l" + +# Set environment variables +ENV DEBIAN_FRONTEND=noninteractive \ + LC_ALL=C.UTF-8 \ + LANG=C.UTF-8 + +# Install build dependencies and Python versions +RUN apt-get update && apt-get install -y \ + # Build essentials + build-essential \ + gcc \ + g++ \ + make \ + cmake \ + git \ + wget \ + curl \ + ca-certificates \ + patchelf \ + # Python build dependencies + libssl-dev \ + libffi-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libgdbm-dev \ + liblzma-dev \ + tk-dev \ + zlib1g-dev \ + # Additional libraries commonly needed for wheels + libxml2-dev \ + libxslt1-dev \ + libyaml-dev \ + libglib2.0-dev \ + # PyGObject dependencies + libcairo2-dev \ + pkg-config \ + libgirepository1.0-dev \ + # dbus-python dependencies + libdbus-1-dev \ + libdbus-glib-1-dev \ + # Pillow dependencies + libjpeg-dev \ + libpng-dev \ + libtiff-dev \ + libfreetype6-dev \ + liblcms2-dev \ + libwebp-dev \ + libopenjp2-7-dev \ + libfribidi-dev \ + libharfbuzz-dev \ + libxcb1-dev \ + libxau-dev \ + brotli \ + libbrotli-dev \ + # Python from Debian repos (Bookworm provides 3.11) + python3 \ + python3-dev \ + python3-venv \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* + +# Install pip and wheel tools +# --break-system-packages is OK because it is docker image and not a system +RUN python3 -m pip install --no-cache-dir --break-system-packages --upgrade \ + pip \ + setuptools \ + wheel \ + auditwheel + +# Set up manylinux platform tag +# This makes auditwheel recognize the environment as manylinux-compatible +# Bookworm has glibc 2.36, so we use manylinux_2_35 +RUN echo "manylinux_2_35_armv7l" > /etc/system-release-cpe + +# Create a marker file for manylinux detection +RUN mkdir -p /opt/python && \ + echo "manylinux_2_35_armv7l" > /opt/python/PLATFORM + +# Set working directory +WORKDIR /workspace + +# Default Python +ENV PATH="/usr/bin:${PATH}" + +CMD ["/bin/bash"] diff --git a/os_dependencies/macos.sh b/os_dependencies/macos.sh index ae3f555..7288b73 100755 --- a/os_dependencies/macos.sh +++ b/os_dependencies/macos.sh @@ -3,7 +3,7 @@ arch=$(uname -m) # PyGObject needs build dependecies https://pygobject.readthedocs.io/en/latest/getting_started.html -brew install pygobject3 gtk4 +brew install pygobject3 gtk4 gobject-introspection # Only MacOS x86_64 additional dependencies diff --git a/os_dependencies/manylinux.sh b/os_dependencies/manylinux.sh new file mode 100644 index 0000000..8686139 --- /dev/null +++ b/os_dependencies/manylinux.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Equivalent to ubuntu.sh and linux_arm.sh but using yum package manager + +# Update package manager +yum update -y + +# PyGObject needs build dependencies +yum install -y cairo-devel pkg-config gobject-introspection-devel + +# dbus-python needs build dependencies +yum install -y cmake dbus-devel dbus-glib-devel + +# Pillow needs image processing libraries +yum install -y \ + libjpeg-devel \ + libpng-devel \ + libtiff-devel \ + zlib-devel \ + freetype-devel \ + lcms2-devel \ + libwebp-devel \ + openjpeg2-devel \ + fribidi-devel \ + harfbuzz-devel \ + libxcb-devel \ + libXau-devel \ + brotli \ + brotli-devel + +# Additional libraries that wheels might depend on +yum install -y \ + gcc \ + gcc-c++ \ + make diff --git a/pyproject.toml b/pyproject.toml index a82e2d2..57dbb7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,6 @@ disallow_incomplete_defs = false # Disallows defining functions with incomplete type annotations disallow_untyped_defs = false # Disallows defining functions without type annotations or with incomplete type annotations ignore_missing_imports = true # Suppress error messages about imports that cannot be resolved - python_version = "3.9" # Specifies the Python version used to parse and check the target program + python_version = "3.8" # Specifies the Python version used to parse and check the target program warn_no_return = true # Shows errors for missing return statements on some execution paths warn_return_any = true # Shows a warning when returning a value with type Any from a function declared with a non- Any return type diff --git a/repair_wheels.py b/repair_wheels.py new file mode 100644 index 0000000..8d1783f --- /dev/null +++ b/repair_wheels.py @@ -0,0 +1,326 @@ +# +# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD +# +# SPDX-License-Identifier: Apache-2.0 +# +""" +Repairs wheels for dynamically linked libraries on all platforms to ensure broad compatibility. +If this is not able to achieve the broken wheel is deleted and not published to Espressif's PyPI. +See: https://github.com/espressif/idf-python-wheels/blob/main/README.md#universal-wheel-tag---linking-of-dynamic-libraries +- Windows: delvewheel (bundles DLLs) +- macOS: delocate (bundles dylibs) +- Linux: auditwheel (bundles SOs) +""" + +import platform +import subprocess + +from pathlib import Path +from typing import Union + +from colorama import Fore + +from _helper_functions import print_color + + +def get_platform() -> str: + return platform.system() + + +def is_pure_python_wheel(wheel_name: str) -> bool: + return "py3-none-any" in wheel_name + + +def is_platform_wheel(wheel_name: str, target_platform: str, current_arch: Union[str, None] = None) -> bool: + """Check if wheel is for the target platform and architecture.""" + if target_platform == "Windows": + return "win" in wheel_name + elif target_platform in ["Darwin", "macOS ARM", "macOS Intel"]: + if "macosx" not in wheel_name: + return False + if target_platform == "macOS ARM" or (target_platform == "Darwin" and current_arch == "arm64"): + return "arm64" in wheel_name or "universal2" in wheel_name + elif target_platform == "macOS Intel" or (target_platform == "Darwin" and current_arch == "x86_64"): + return "x86_64" in wheel_name or "universal2" in wheel_name + return True # If no specific arch check needed + elif target_platform == "Linux": + return "linux" in wheel_name + return False + + +def get_wheel_arch(wheel_name: str) -> Union[str, None]: + """Extract architecture from wheel filename.""" + # {name}-{version}-{python_tag}-{abi_tag}-{platform_tag}.whl + parts = wheel_name.replace(".whl", "").split("-") + if len(parts) >= 5: + platform_tag = parts[-1] + for arch in ["x86_64", "aarch64", "armv7l"]: + if arch in platform_tag: + return arch + return None + + +def repair_wheel_windows(wheel_path: Path, temp_dir: Path) -> subprocess.CompletedProcess[str]: + """Repair Windows wheel using delvewheel.""" + result = subprocess.run( + ["delvewheel", "repair", str(wheel_path), "-w", str(temp_dir), "--no-mangle-all"], + capture_output=True, + text=True, + ) + return result + + +def fix_universal2_wheel_name(wheel_path: Path, error_msg: str) -> Union[Path, str, None]: + """Fix incorrectly tagged universal2 wheel by renaming to actual architecture. + + Returns: + - New wheel path if successfully renamed + - None if not a fixable case + - "delete" if wheel is corrupted (missing all architectures) and should be deleted + """ + if ( + "universal2" not in str(wheel_path) + or "Failed to find any binary with the required architecture" not in error_msg + ): + return None + + # Parse which architecture is missing from the error + # "Failed to find any binary with the required architecture: 'x86_64'" + # means the wheel only has arm64 + if "'arm64,x86_64'" in error_msg or "'x86_64,arm64'" in error_msg: + # Missing BOTH architectures - wheel is corrupted, delete it + print_color(" -> Deleting corrupted wheel (missing native binaries for all architectures)", Fore.RED) + wheel_path.unlink() + return "delete" + elif "'x86_64'" in error_msg: + # Missing x86_64, so it only has arm64 + actual_arch = "arm64" + elif "'arm64'" in error_msg: + # Missing arm64, so it only has x86_64 + actual_arch = "x86_64" + else: + return None + + # Rename the wheel + new_name = str(wheel_path.name).replace("_universal2.whl", f"_{actual_arch}.whl") + new_path = wheel_path.parent / new_name + + print(f" -> Renaming: {wheel_path.name} -> {new_name}") + wheel_path.rename(new_path) + + return new_path + + +def repair_wheel_macos(wheel_path: Path, temp_dir: Path) -> subprocess.CompletedProcess[str]: + """Repair macOS wheel using delocate.""" + cmd = ["delocate-wheel", "-w", str(temp_dir), "-v", str(wheel_path)] + result = subprocess.run(cmd, capture_output=True, text=True) + return result + + +def repair_wheel_linux(wheel_path: Path, temp_dir: Path) -> subprocess.CompletedProcess[str]: + """Repair Linux wheel using auditwheel.""" + result = subprocess.run( + ["auditwheel", "repair", str(wheel_path), "-w", str(temp_dir)], capture_output=True, text=True + ) + return result + + +def main() -> None: + wheels_dir: Path = Path("./downloaded_wheels") + temp_dir: Path = Path("./temp_repair") + temp_dir.mkdir(exist_ok=True) + + # Find all wheel files + wheels: list[Path] = list(wheels_dir.rglob("*.whl")) + + if not wheels: + print_color(f"No wheels found in {wheels_dir}", Fore.RED) + raise SystemExit("No wheels found in downloaded_wheels directory") + + print_color(f"Found {len(wheels)} wheels\n") + + current_platform: str = get_platform() + current_arch: str = platform.machine() + + repaired_count: int = 0 + skipped_count: int = 0 + deleted_count: int = 0 + error_count: int = 0 + errors: list[str] = [] + + # Repair each wheel + for wheel in wheels: + print(f"Processing: {wheel.name}") + + # Skip pure Python wheels + if is_pure_python_wheel(wheel.name): + print_color(" -> Skipping pure Python wheel") + skipped_count += 1 + continue + + # Skip pywin32 wheels on Windows (DLLs are internal to the wheel) + if current_platform == "Windows" and wheel.name.startswith("pywin32"): + print_color(" -> Skipping pywin32 wheel (DLLs are internal)") + skipped_count += 1 + continue + + # Skip wheels not for the workflow platform + if not is_platform_wheel(wheel.name, current_platform, current_arch): + print_color(f" -> Skipping (not a {current_platform} wheel)") + skipped_count += 1 + continue + + # For Linux, skip wheels for different architectures + if current_platform == "Linux": + wheel_arch = get_wheel_arch(wheel.name) + if wheel_arch and wheel_arch != current_arch: + print_color(f" -> Skipping incompatible architecture ({wheel_arch} wheel on {current_arch} platform)") + skipped_count += 1 + continue + + # Clean temp directory + for old_wheel in temp_dir.glob("*.whl"): + old_wheel.unlink() + + # Repair wheel using platform-specific tool + if current_platform == "Windows": + result = repair_wheel_windows(wheel, temp_dir) + elif current_platform == "Darwin": + result = repair_wheel_macos(wheel, temp_dir) + elif current_platform == "Linux": + result = repair_wheel_linux(wheel, temp_dir) + else: + print_color(f" -> ERROR: Unsupported platform {current_platform}", Fore.RED) + error_count += 1 + continue + + if result.stdout: + print(f" {result.stdout.strip()}") + if result.stderr: + print_color(f" {result.stderr.strip()}", Fore.RED) + + # Check for errors + error_msg = result.stderr.strip() if result.stderr else "" + + # Special handling for incorrectly tagged universal2 wheels on macOS + if ( + current_platform == "Darwin" + and "universal2" in wheel.name + and "Failed to find any binary with the required architecture" in error_msg + ): + # Try to fix by renaming the wheel to the correct architecture + renamed_wheel = fix_universal2_wheel_name(wheel, error_msg) + + if renamed_wheel == "delete": + # Wheel was corrupted and has been deleted + deleted_count += 1 + continue + elif renamed_wheel: + # Clean temp directory and retry with renamed wheel + for old_wheel in temp_dir.glob("*.whl"): + old_wheel.unlink() + + print_color(" -> Retrying delocate with corrected wheel name", Fore.CYAN) + result = repair_wheel_macos(Path(renamed_wheel), temp_dir) + + if result.stdout: + print(f" {result.stdout.strip()}") + if result.stderr: + print_color(f" {result.stderr.strip()}", Fore.RED) + + # Update wheel reference and error message for subsequent checks + wheel = Path(renamed_wheel) + error_msg = result.stderr.strip() if result.stderr else "" + + # Special handling forLinux ARMv7 broken wheels + if ( + current_platform == "Linux" + and current_arch == "armv7l" + and "This does not look like a platform wheel, no ELF executable" in error_msg + ): + print_color(" -> Deleting corrupted wheel", Fore.RED) + wheel.unlink() + deleted_count += 1 + continue + + # Check for non-critical errors (keep original wheel) + is_noncritical = ( + "too-recent versioned symbols" in error_msg + # manylinux wheel can't find its libraries + # it means it was already properly repaired + or ("manylinux" in wheel.name and "could not be located" in error_msg) + ) + + has_error = ( + any( + [ + "ValueError:" in error_msg, + "FileNotFoundError:" in error_msg, + "Cannot repair wheel" in error_msg, + "could not be located" in error_msg, + "DelocationError:" in error_msg, + ] + ) + and not is_noncritical + ) + + if is_noncritical: + # Non-critical error - keep the wheel + if "too-recent versioned symbols" in error_msg: + print_color(" -> Keeping original wheel (build issue: needs older toolchain)", Fore.YELLOW) + elif "manylinux" in wheel.name and "could not be located" in error_msg: + print_color(" -> Keeping original wheel (already bundled from PyPI)", Fore.GREEN) + skipped_count += 1 + elif has_error: + # Actual error occurred (even if a wheel was created, it may be broken) + # Clean up any partial wheel + for old_wheel in temp_dir.glob("*.whl"): + old_wheel.unlink() + print_color(f" -> ERROR: {error_msg}", Fore.RED) + errors.append(f"{wheel.name}: {error_msg}") + error_count += 1 + else: + # Check if a repaired wheel was created + repaired = next(temp_dir.glob("*.whl"), None) + + if repaired: + # A repaired wheel was created successfully + if repaired.name != wheel.name: + wheel.unlink() # Remove original + repaired.rename(wheel.parent / repaired.name) + print_color(f" -> Replaced with repaired wheel: {repaired.name}", Fore.GREEN) + else: + # Name unchanged + wheel.unlink() + repaired.rename(wheel) + print_color(f" -> Repaired successfully: {repaired.name}", Fore.GREEN) + repaired_count += 1 + elif result.returncode == 0: + # No repaired wheel created, but command succeeded (already compatible) + print_color(" -> Keeping original wheel (already compatible)", Fore.GREEN) + skipped_count += 1 + else: + # Command failed and no wheel created + print_color(f" -> ERROR: {error_msg}", Fore.RED) + errors.append(f"{wheel.name}: {error_msg}") + error_count += 1 + + print_color("---------- STATISTICS ----------") + print_color(f"Total wheels: {len(wheels)}") + print_color(f"Deleted wheels: {deleted_count}", Fore.RED) + print_color(f"Kept wheels: {skipped_count}") + print_color(f"Repaired wheels: {repaired_count}", Fore.GREEN) + print_color(f"Errors: {error_count}", Fore.RED) + + if errors: + print_color("---------- ERRORS ----------", Fore.RED) + for i, error in enumerate(errors, start=1): + print_color(f"{i}. {error}", Fore.RED) + raise SystemExit("One or more wheels failed to repair") + + print("All wheels processed successfully") + + +if __name__ == "__main__": + main() diff --git a/resources/idf-python-wheels_diagram_source.txt b/resources/idf-python-wheels-diagram-source.txt similarity index 100% rename from resources/idf-python-wheels_diagram_source.txt rename to resources/idf-python-wheels-diagram-source.txt diff --git a/resources/idf-python-wheels_diagram.svg b/resources/idf-python-wheels-diagram.svg similarity index 66% rename from resources/idf-python-wheels_diagram.svg rename to resources/idf-python-wheels-diagram.svg index ef9d97a..84fc8d7 100644 --- a/resources/idf-python-wheels_diagram.svg +++ b/resources/idf-python-wheels-diagram.svg @@ -1,14 +1,14 @@ + inkscape:document-units="mm" + showgrid="false" /> + ry="22.658775" + sodipodi:insensitive="true" /> + width="639.8222" + height="1130.1211" + x="2.6426644" + y="22.1595" + rx="12.645135" + ry="15.247279" + sodipodi:insensitive="true" /> Build wheels for IDF (build_wheels.py) @@ -414,7 +417,7 @@ transform="matrix(0.66925281,0,0,0.66925281,176.46167,-141.91629)">- print statistics @@ -674,10 +677,15 @@ x="126" y="1618.7559" id="text44" - transform="matrix(0.66925281,0,0,0.66925281,377.23751,-270.70933)">upload-python-wheels.yml + transform="matrix(0.55461667,0,0,0.66925281,399.80593,-270.70933)" + style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:12px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, ';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-variant-east-asian:normal;fill:#000000">manylinux-repair.yml Download artifacts + transform="matrix(0.66925281,0,0,0.66925281,185.0223,-99.61521)">Download artifacts - wheels directories + transform="matrix(0.66925281,0,0,0.66925281,185.0223,-99.61521)">- wheels directories Upload to S3 + transform="matrix(0.66925281,0,0,0.66925281,185.0223,-99.61521)">Upload to S3 - upload_wheels.py + transform="matrix(0.66925281,0,0,0.66925281,185.0223,-99.61521)">- upload_wheels.py - create_index_pages.py + transform="matrix(0.66925281,0,0,0.66925281,185.0223,-99.61521)">- create_index_pages.py @@ -788,7 +796,7 @@ id="line52" /> @@ -799,7 +807,7 @@ sodipodi:nodetypes="cc" /> @@ -812,7 +820,7 @@ id="line54" /> @@ -825,7 +833,7 @@ id="line55" /> @@ -838,7 +846,7 @@ id="line56" /> @@ -851,7 +859,7 @@ id="line57" /> @@ -864,7 +872,7 @@ id="line58" /> @@ -877,7 +885,7 @@ id="line59" /> @@ -895,7 +903,7 @@ sodipodi:nodetypes="cc" /> @@ -957,7 +965,7 @@ id="line59-4" /> @@ -968,7 +976,7 @@ sodipodi:nodetypes="cc" /> @@ -979,7 +987,7 @@ sodipodi:nodetypes="cc" /> @@ -992,7 +1000,7 @@ id="line65" /> @@ -1005,7 +1013,7 @@ id="line71" /> @@ -1018,10 +1026,128 @@ id="line72" /> + + + 1 or more wheel(s) repair failed + + Raise Error + + + yes + + + + + + + + @@ -1044,7 +1170,7 @@ id="line74" /> @@ -1057,7 +1183,7 @@ id="line76" /> @@ -1070,7 +1196,7 @@ id="line77" /> @@ -1083,7 +1209,7 @@ id="line78" /> @@ -1096,49 +1222,79 @@ id="line79" /> + + Upload artifacts + - wheels directories + transform="matrix(0.66925281,0,0,0.66925281,185.0223,-99.61521)" /> + transform="matrix(0.66925281,0,0,0.66925281,185.0223,-99.61521)" /> + transform="matrix(0.66925281,0,0,0.66925281,185.0223,-99.61521)" /> + transform="matrix(0,0.66925281,-0.66925281,0,582.70631,672.56585)" /> + + + + upload-python-wheels.yml + + + Download artifacts + - wheels directories + + Repair wheels + - run wheel repair tool + - change the repaired wheels + + + + + + diff --git a/update_python_versions.py b/update_python_versions.py index 57eb874..a8b5677 100644 --- a/update_python_versions.py +++ b/update_python_versions.py @@ -61,7 +61,7 @@ def update_python_versions(): ruff_pattern = r'(target-version\s*=\s*["\'])py\d+(["\'])' new_pyproject = re.sub(ruff_pattern, f"\\g<1>{py_version_compact}\\g<2>", pyproject_content) - # Update mypy python_version (e.g., "3.8") + # Update mypy python_version (e.g., "3.9") mypy_pattern = r'(python_version\s*=\s*["\'])\d+\.\d+(["\'])' new_pyproject = re.sub(mypy_pattern, f"\\g<1>{oldest_python}\\g<2>", new_pyproject)