From 59dc271467a30865aaf447ba94838ae879c5b642 Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 16 Dec 2025 17:44:45 +0200 Subject: [PATCH 1/2] cleanup and single-use OPENBLAS_COMMIT via a file --- .github/PULL_REQUEST_TEMPLATE.md | 4 +- .github/workflows/posix.yml | 9 -- .github/workflows/windows-arm.yml | 1 - .github/workflows/windows.yml | 3 +- .gitmodules | 3 - OpenBLAS | 2 +- README.md | 4 +- build-openblas.sh | 57 +----------- ci-before-build.sh | 5 +- gfortran-install | 1 - github_commit.txt | 1 + openblas_commit.txt | 1 + pyproject.toml | 2 +- tools/build_prepare.sh | 5 +- tools/build_steps.sh | 93 +++---------------- tools/build_steps_win_arm64.bat | 11 +-- ...ild_openblas.sh => build_steps_windows.sh} | 9 +- tools/build_wheel.sh | 4 +- tools/docker_build_wrap.sh | 13 --- 19 files changed, 38 insertions(+), 190 deletions(-) delete mode 160000 gfortran-install create mode 100644 github_commit.txt create mode 100644 openblas_commit.txt rename tools/{build_openblas.sh => build_steps_windows.sh} (96%) delete mode 100755 tools/docker_build_wrap.sh diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index f73235c5..d9d6850f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1 +1,3 @@ -- [ ] I updated the package version in pyproject.toml and made sure the first 3 numbers match `git describe --tags --abbrev=8` in OpenBLAS at the `OPENBLAS_COMMIT`. If I did not update `OPENBLAS_COMMIT`, I incremented the wheel build number (i.e. 0.3.29.0.0 to 0.3.29.0.1) +- [ ] I updated the package version in pyproject.toml and made sure the first 3 numbers match `./openblas_commit.txt`. If I did not update `./openblas_commit.txt`, I incremented the wheel build number (i.e. 0.3.29.0.0 to 0.3.29.0.1) + +Note: update `./openblas_commit.txt` with `cd OpenBLAS; git describe --tags --abbrev=8 > ../openblas_commit.txt` diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 950ad2ed..6d4fe645 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -11,9 +11,6 @@ on: schedule: - cron: '0 0 * * 0' -env: - OPENBLAS_COMMIT: "v0.3.30-359-g29fab2b9" - jobs: build: runs-on: ${{ matrix.os }} @@ -118,12 +115,6 @@ jobs: path: dist/scipy_openblas*.whl - - name: Upload openblas to artifacts - uses: actions/upload-artifact@v4.3.0 - with: - name: openblas-${{ matrix.os }}-${{ matrix.PLAT }}-${{ matrix.INTERFACE64 }}-${{ matrix.MB_ML_LIBC }}-${{ matrix.MB_ML_VER }} - path: dist/openblas*.tar.gz - - uses: conda-incubator/setup-miniconda@v3.2.0 with: channels: conda-forge diff --git a/.github/workflows/windows-arm.yml b/.github/workflows/windows-arm.yml index bb98bb1a..01c66e21 100644 --- a/.github/workflows/windows-arm.yml +++ b/.github/workflows/windows-arm.yml @@ -9,7 +9,6 @@ on: - maintenance/** env: - OPENBLAS_COMMIT: "v0.3.30-359-g29fab2b9" OPENBLAS_ROOT: "c:\\opt" # Preserve working directory for calls into bash # Without this, invoking bash will cd to the home directory diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index d17f57e4..2fab5137 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -10,7 +10,6 @@ on: workflow_dispatch: null env: - OPENBLAS_COMMIT: "v0.3.30-359-g29fab2b9" OPENBLAS_ROOT: "c:\\opt" # Preserve working directory for calls into bash # Without this, invoking bash will cd to the home directory @@ -71,7 +70,7 @@ jobs: - name: Build run: | git submodule update --init --recursive - & $env:BASH_PATH -lc tools/build_openblas.sh + & $env:BASH_PATH -lc tools/build_steps_windows.sh - name: Test run: | diff --git a/.gitmodules b/.gitmodules index 7b8f5ca3..546361d9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "OpenBLAS"] path = OpenBLAS url = https://github.com/xianyi/OpenBLAS.git -[submodule "gfortran-install"] - path = gfortran-install - url = https://github.com/MacPython/gfortran-install.git diff --git a/OpenBLAS b/OpenBLAS index 29fab2b9..5aff62eb 160000 --- a/OpenBLAS +++ b/OpenBLAS @@ -1 +1 @@ -Subproject commit 29fab2b912d3130b3caa84c2c1178f94ed25d1a6 +Subproject commit 5aff62eb96c40d1c8fb6d7c5b71a946be8842970 diff --git a/README.md b/README.md index f1ba0ba2..650b06dc 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ # OpenBLAS library build process -First, tarballs are built using `do_build_lib` in `tools/build_steps.sh` (on -posix in a docker and drectly on macos) or `build_openblas.sh` on windows. +First, tarballs are built using `build_lib` in `tools/build_steps.sh` (on +posix in a docker and drectly on macos) or `tools/build_steps_windows.sh` on windows. Then the shared object and header files from the tarball are used to build the wheel via `tools/build_wheel.sh`, and the wheels uploaded to diff --git a/build-openblas.sh b/build-openblas.sh index f50b00e0..427f6d3f 100755 --- a/build-openblas.sh +++ b/build-openblas.sh @@ -1,60 +1,11 @@ #! /bin/bash - -# Most of the content in this file comes from https://github.com/multi-build/multibuild, with some modifications -# Follow the license below - - - -# .. _license: - -# ********************* -# Copyright and License -# ********************* - -# The multibuild package, including all examples, code snippets and attached -# documentation is covered by the 2-clause BSD license. - -# Copyright (c) 2013-2024, Matt Terry and Matthew Brett; all rights -# reserved. - -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: - -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. - -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS -# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - set -xeo pipefail source tools/build_steps.sh echo "------ BEFORE BUILD ---------" before_build -if [[ "$NIGHTLY" = "true" ]]; then - echo "------ CLEAN CODE --------" - clean_code develop - echo "------ BUILD LIB --------" - build_lib "$PLAT" "$INTERFACE64" "1" -else - echo "------ CLEAN CODE --------" - clean_code $OPENBLAS_COMMIT - echo "------ BUILD LIB --------" - build_lib "$PLAT" "$INTERFACE64" "0" -fi +echo "------ CLEAN CODE --------" +clean_code $OPENBLAS_COMMIT +echo "------ BUILD LIB --------" +build_lib "$PLAT" "$INTERFACE64" diff --git a/ci-before-build.sh b/ci-before-build.sh index d356dc31..83fbe493 100755 --- a/ci-before-build.sh +++ b/ci-before-build.sh @@ -44,10 +44,13 @@ set -xe if [[ "$NIGHTLY" = "true" ]]; then pushd OpenBLAS git checkout develop + export OPENBLAS_COMMIT=$(git describe --tags --abbrev=8) # Set the pyproject.toml version: convert v0.3.24-30-g138ed79f to 0.3.34.30 - version=$(git describe --tags --abbrev=8 | sed -e "s/^v\(.*\)-g.*/\1/" | sed -e "s/-/./g") + version=$(echo $OPENBLAS_COMMIT | sed -e "s/^v\(.*\)-g.*/\1/" | sed -e "s/-/./g") popd sed -e "s/^version = .*/version = \"${version}\"/" -i.bak pyproject.toml +else + export OPENBLAS_COMMIT=$(cat openblas_commit.txt) fi if [ "$(uname)" != "Darwin" ]; then diff --git a/gfortran-install b/gfortran-install deleted file mode 160000 index 3dd38d9c..00000000 --- a/gfortran-install +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3dd38d9ce78b3890598cb0eff18a7bec50c06f5e diff --git a/github_commit.txt b/github_commit.txt new file mode 100644 index 00000000..3ee68f2b --- /dev/null +++ b/github_commit.txt @@ -0,0 +1 @@ +v0.3.30-392-g5aff62eb diff --git a/openblas_commit.txt b/openblas_commit.txt new file mode 100644 index 00000000..4e0cc642 --- /dev/null +++ b/openblas_commit.txt @@ -0,0 +1 @@ +v0.3.30-359-g29fab2b9 diff --git a/pyproject.toml b/pyproject.toml index aae8c916..c538cb82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "scipy-openblas64" # v0.3.30-359-g29fab2b9 -version = "0.3.30.359.0" +version = "0.3.30.359.1" requires-python = ">=3.7" description = "Provides OpenBLAS for python packaging" readme = "README.md" diff --git a/tools/build_prepare.sh b/tools/build_prepare.sh index cb99d44c..dbfc303d 100644 --- a/tools/build_prepare.sh +++ b/tools/build_prepare.sh @@ -13,8 +13,7 @@ mkdir -p local/openblas mkdir -p dist $PYTHON -m pip install wheel auditwheel -# This will fail if there is more than one file in libs -tar -C local/scipy_openblas64 --strip-components=2 -xf libs/openblas*.tar.gz +tar -C local/scipy_openblas64 --strip-components=2 -xf libs/openblas.tar.gz # do not package the static libs and symlinks, only take the shared object find local/scipy_openblas64/lib -maxdepth 1 -type l -delete @@ -69,4 +68,4 @@ fi rm -rf dist/* -echo "The build preparation is done." \ No newline at end of file +echo "The build preparation is done." diff --git a/tools/build_steps.sh b/tools/build_steps.sh index e21e7a2c..448d732f 100644 --- a/tools/build_steps.sh +++ b/tools/build_steps.sh @@ -8,7 +8,7 @@ ROOT_DIR=$(dirname $(dirname "${BASH_SOURCE[0]}")) MB_PYTHON_VERSION=3.9 function before_build { - # Manylinux Python version set in build_lib + # install gfortran, objconv on macOS if [ "$(uname -s)" == "Darwin" ]; then if [ ! -e /usr/local/lib ]; then sudo mkdir -p /usr/local/lib @@ -86,25 +86,6 @@ function get_plat_tag { echo "macosx_${target}_${plat}" } -function build_lib { - # OSX or manylinux build - # - # Input arg - # plat - one of i686, x86_64, arm64 - # interface64 - 1 if build with INTERFACE64 and SYMBOLSUFFIX - # nightly - 1 if building for nightlies - # - # Depends on globals - # BUILD_PREFIX - install suffix e.g. "/usr/local" - # MB_ML_VER - set -x - local plat=${1:-$PLAT} - local interface64=${2:-$INTERFACE64} - local nightly=${3:0} - local manylinux=${MB_ML_VER:-1} - do_build_lib "$plat" "$interface64" "$nightly" -} - function patch_source { # Runs inside OpenBLAS directory # Make the patches by git format-patch @@ -114,19 +95,21 @@ function patch_source { done } -function do_build_lib { - # Build openblas lib +function build_lib { + # OSX or manylinux build + # # Input arg # plat - one of i686, x86_64, arm64 - # interface64 (optional) - whether to build ILP64 openblas - # with 64_ symbol suffix - # nightly (optional) - whether to build for nightlies + # interface64 - 1 if build with INTERFACE64 and SYMBOLSUFFIX # # Depends on globals # BUILD_PREFIX - install suffix e.g. "/usr/local" - local plat=$1 - local interface64=$2 - local nightly=$3 + # MB_ML_VER + + set -x + local plat=${1:-$PLAT} + local interface64=${2:-$INTERFACE64} + case $(uname -s)-$plat in Linux-x86_64) local bitness=64 @@ -223,11 +206,6 @@ function do_build_lib { fi make PREFIX=$BUILD_PREFIX $interface_flags install popd - if [ "$nightly" = "1" ]; then - local version="HEAD" - else - local version=$(cd OpenBLAS && git describe --tags --abbrev=8) - fi mv $BUILD_PREFIX/lib/pkgconfig/openblas*.pc $BUILD_PREFIX/lib/pkgconfig/scipy-openblas.pc local plat_tag=$(get_plat_tag $plat) if [ "$interface64" = "1" ]; then @@ -242,7 +220,7 @@ function do_build_lib { rm $BUILD_PREFIX/lib/pkgconfig/scipy-openblas.pc.bak fi - local out_name="openblas${symbolsuffix}-${version}-${plat_tag}.tar.gz" + local out_name="openblas.tar.gz" tar zcvf libs/$out_name \ $BUILD_PREFIX/include/*blas* \ $BUILD_PREFIX/include/*lapack* \ @@ -250,50 +228,3 @@ function do_build_lib { $BUILD_PREFIX/lib/pkgconfig/scipy-openblas* \ $BUILD_PREFIX/lib/cmake/openblas } - - -function build_lib_on_travis { - # OSX or manylinux build - # - # Input arg - # plat - one of i686, x86_64, arm64 - # interface64 - 1 if build with INTERFACE64 and SYMBOLSUFFIX - # nightly - 1 if building for nightlies - # - # Depends on globals - # BUILD_PREFIX - install suffix e.g. "/usr/local" - # MB_ML_VER - set -x - local plat=${1:-$PLAT} - local interface64=${2:-$INTERFACE64} - local nightly=${3:0} - local manylinux=${MB_ML_VER:-1} - - # Manylinux wrapper - local libc=${MB_ML_LIBC:-manylinux} - local docker_image=quay.io/pypa/${libc}${manylinux}_${plat} - docker pull $docker_image - # run `do_build_lib` in the docker image - docker run --rm \ - -e BUILD_PREFIX="$BUILD_PREFIX" \ - -e PLAT="${plat}" \ - -e INTERFACE64="${interface64}" \ - -e NIGHTLY="${nightly}" \ - -e PYTHON_VERSION="$MB_PYTHON_VERSION" \ - -e MB_ML_VER=${manylinux} \ - -e MB_ML_LIBC=${libc} \ - -v $PWD:/io \ - $docker_image /io/tools/docker_build_wrap.sh -} - - - -function build_on_travis { - if [ ${TRAVIS_EVENT_TYPE} == "cron" ]; then - build_lib_on_travis "$PLAT" "$INTERFACE64" 1 - version=$(cd OpenBLAS && git describe --tags --abbrev=8 | sed -e "s/^v\(.*\)-g.*/\1/" | sed -e "s/-/./g") - sed -e "s/^version = .*/version = \"${version}\"/" -i.bak pyproject.toml - else - build_lib_on_travis "$PLAT" "$INTERFACE64" 0 - fi -} diff --git a/tools/build_steps_win_arm64.bat b/tools/build_steps_win_arm64.bat index f53d9578..21c997c4 100755 --- a/tools/build_steps_win_arm64.bat +++ b/tools/build_steps_win_arm64.bat @@ -8,11 +8,6 @@ :: If INTERFACE64 environment variable is 1, then if_bits defaults to 64 :: Expects these binaries on the PATH: :: clang-cl, flang-new, cmake, perl -:: Uses environment variable: -:: OPENBLAS_COMMIT (unspec -> current submodule commit, if contains -:: Windows on ARM build fixes (see below), otherwise earliest commit -:: with those fixes). - :: First commit containing WoA build fixes. :: Minimum OpenBLAS commit to build; we'll update to this if commit not :: present. @@ -63,13 +58,11 @@ if "%if_bits%"=="64" ( echo Cloning OpenBLAS repository with submodules... git submodule update --init --recursive OpenBLAS if errorlevel 1 exit /b 1 +set /p OPENBLAS_COMMIT=NUL if errorlevel 1 ( echo OpenBLAS commit does not contain WoA build fixes. diff --git a/tools/build_openblas.sh b/tools/build_steps_windows.sh similarity index 96% rename from tools/build_openblas.sh rename to tools/build_steps_windows.sh index 71e0d24c..5ae3cc36 100644 --- a/tools/build_openblas.sh +++ b/tools/build_steps_windows.sh @@ -1,9 +1,9 @@ #!/bin/bash # Build script for OpenBLAS on Windows # -# Usage: build_openblas.sh [openblas_root [build_bits [if_bits]]] +# Usage: build_steps_windows.sh [openblas_root [build_bits [if_bits]]] # -# e.g build_openblas.sh c:\\opt 64 32 +# e.g build_steps_windows.sh c:\\opt 64 32 # # Uses the optional environment variables. We always prefer command line argument # values above to environment variable values: @@ -13,7 +13,6 @@ # INTERFACE64 (1 for 64-bit interface, anything else or undefined for 32, # This gives the default value if if_bits not specified above). # START_DIR (directory containing OpenBLAS source, unspec -> .. from here) -# OPENBLAS_COMMIT (unspec -> current submodule commit) # LDFLAGS (example: "-lucrt -static -static-libgcc") # # Expects at leasts these binaries on the PATH: @@ -50,9 +49,7 @@ git submodule update --init --recursive # Get / clean code git fetch origin -if [ -n "$OPENBLAS_COMMIT" ]; then - git checkout $OPENBLAS_COMMIT -fi +git checkout $(cat ../openblas_commit.txt) git clean -fxd git reset --hard rm -rf $openblas_root/$build_bits diff --git a/tools/build_wheel.sh b/tools/build_wheel.sh index 394dcb68..cb3aa921 100644 --- a/tools/build_wheel.sh +++ b/tools/build_wheel.sh @@ -3,8 +3,6 @@ # $INTERFACE64 ("1" or "0") # $PLAT (x86_64, i686, arm64, aarch64, s390x, ppc64le) -# The code below is for Travis use only. - set -xe if [[ ! -e tools/build_prepare.sh ]];then @@ -51,4 +49,4 @@ else $PYTHON -m scipy_openblas64 fi -echo "Wheel test successful." \ No newline at end of file +echo "Wheel test successful." diff --git a/tools/docker_build_wrap.sh b/tools/docker_build_wrap.sh deleted file mode 100755 index 4522340e..00000000 --- a/tools/docker_build_wrap.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Depends on: -# PLAT -# INTERFACE64 (could be missing or empty) -# NIGHTLY (could be missing or empty) -set -e - -# Change into root directory of repo -if [[ ! -e tools/build_steps.sh ]];then - cd /io -fi -source tools/build_steps.sh -do_build_lib "$PLAT" "$INTERFACE64" "$NIGHTLY" From 28aad696c21544f631348e718b8b98aca2b42d13 Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 16 Dec 2025 18:04:34 +0200 Subject: [PATCH 2/2] do not upload openblas zip --- .github/workflows/windows.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 2fab5137..bbd61d9c 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -147,11 +147,6 @@ jobs: name: wheels-${{matrix.plat }}-${{ matrix.INTERFACE64 }} path: dist/scipy_openblas*.whl - - uses: actions/upload-artifact@v4.3.0 - with: - name: openblas-${{matrix.plat }}-${{ matrix.INTERFACE64 }} - path: builds/openblas*.zip - - name: Test 64-bit interface wheel if: matrix.INTERFACE64 == '1' run: |