From c311deaaf1c67378f2df300b2ffd7e3431cb8cf3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 12 Mar 2025 10:47:33 -0500 Subject: [PATCH 1/6] PYTHON-5203 Use uv from Python toolchain if available --- .evergreen/scripts/configure-env.sh | 19 +++++++++++++++++++ .evergreen/scripts/install-dependencies.sh | 10 +++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index f23af8a811..d53523ca06 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -29,6 +29,25 @@ fi PATH_EXT="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PYMONGO_BIN_DIR:\$PATH" +# If the toolchain is available, symlink binaries to the bin dir. +_bin_path="" +if [ "Windows_NT" == "${OS:-}" ]; then + _bin_path="/cygdrive/c/Python/Current/Scripts" +elif [ "$(uname -s)" != "Darwin" ]; then + _bin_path="/Library/Frameworks/Python.Framework/Versions/Current/bin" +else + _bin_path="/opt/python/Current/bin" +fi +if [ -d "${_bin_path}" ]; then + _suffix="" + if [ "Windows_NT" == "${OS:-}" ]; then + _suffix=".exe" + fi + ln -s ${_bin_path}/just${_suffix} $PYMONGO_BIN_DIR/just + ln -s ${_bin_path}/uv${_suffix} $PYMONGO_BIN_DIR/uv + ln -s ${_bin_path}/uvx${_suffix} $PYMONGO_BIN_DIR/uvx +fi + # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh index 31ae4c1735..5ec06a87df 100755 --- a/.evergreen/scripts/install-dependencies.sh +++ b/.evergreen/scripts/install-dependencies.sh @@ -24,10 +24,14 @@ function _pip_install() { echo "Installing $2 using pip..." createvirtualenv "$(find_python3)" $_VENV_PATH python -m pip install $1 + _suffix="" if [ "Windows_NT" = "${OS:-}" ]; then - ln -s "$(which $2)" $_BIN_DIR/$2.exe - else - ln -s "$(which $2)" $_BIN_DIR/$2 + _suffix=".exe" + fi + ln -s "$(which $2)" $_BIN_DIR/${2}${_suffix} + # uv also comes with a uvx binary. + if [ $2 == "uv" ]; then + ln -s "$(which uvx)" $_BIN_DIR/uvx${_suffix} fi echo "Installed to ${_BIN_DIR}" echo "Installing $2 using pip... done." From 6335f9cd405cb1d74b35afd67de02ba2cc9bd1ea Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 12 Mar 2025 11:04:17 -0500 Subject: [PATCH 2/6] fix windows handling --- .evergreen/scripts/configure-env.sh | 30 ++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index d53523ca06..2f47a9d265 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -29,6 +29,18 @@ fi PATH_EXT="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PYMONGO_BIN_DIR:\$PATH" +# Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory +if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin + DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) + PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY) + CARGO_HOME=$(cygpath -m $CARGO_HOME) + UV_TOOL_DIR=$(cygpath -m "$UV_TOOL_DIR") + UV_CACHE_DIR=$(cygpath -m "$UV_CACHE_DIR") + DRIVERS_TOOLS_BINARIES=$(cygpath -m "$DRIVERS_TOOLS_BINARIES") + MONGODB_BINARIES=$(cygpath -m "$MONGODB_BINARIES") + PYMONGO_BIN_DIR=$(cygpath -m "$PYMONGO_BIN_DIR") +fi + # If the toolchain is available, symlink binaries to the bin dir. _bin_path="" if [ "Windows_NT" == "${OS:-}" ]; then @@ -43,21 +55,9 @@ if [ -d "${_bin_path}" ]; then if [ "Windows_NT" == "${OS:-}" ]; then _suffix=".exe" fi - ln -s ${_bin_path}/just${_suffix} $PYMONGO_BIN_DIR/just - ln -s ${_bin_path}/uv${_suffix} $PYMONGO_BIN_DIR/uv - ln -s ${_bin_path}/uvx${_suffix} $PYMONGO_BIN_DIR/uvx -fi - -# Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory -if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin - DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) - PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY) - CARGO_HOME=$(cygpath -m $CARGO_HOME) - UV_TOOL_DIR=$(cygpath -m "$UV_TOOL_DIR") - UV_CACHE_DIR=$(cygpath -m "$UV_CACHE_DIR") - DRIVERS_TOOLS_BINARIES=$(cygpath -m "$DRIVERS_TOOLS_BINARIES") - MONGODB_BINARIES=$(cygpath -m "$MONGODB_BINARIES") - PYMONGO_BIN_DIR=$(cygpath -m "$PYMONGO_BIN_DIR") + ln -s ${_bin_path}/just${_suffix} $PYMONGO_BIN_DIR/just${_suffix} + ln -s ${_bin_path}/uv${_suffix} $PYMONGO_BIN_DIR/uv${_suffix} + ln -s ${_bin_path}/uvx${_suffix} $PYMONGO_BIN_DIR/uvx${_suffix} fi SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts" From 93777a431aa210762fd834c17a866ebffd89a6f6 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 12 Mar 2025 13:06:42 -0500 Subject: [PATCH 3/6] try with fork --- .evergreen/scripts/configure-env.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 2f47a9d265..7efabec0f3 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -97,6 +97,8 @@ EOT rm -rf $DRIVERS_TOOLS BRANCH=master ORG=mongodb-labs +BRANCH=PYTHON-5203 +ORG=blink1073 git clone --branch $BRANCH https://github.com/$ORG/drivers-evergreen-tools.git $DRIVERS_TOOLS cat < ${DRIVERS_TOOLS}/.env From a4d4b4c5d4328a136460323da24394330b8571a1 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 12 Mar 2025 13:27:42 -0500 Subject: [PATCH 4/6] debug --- .evergreen/scripts/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index dcb50cc4dc..ef98b16470 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -136,5 +136,10 @@ def run_command(cmd: str | list[str], **kwargs: Any) -> None: cmd = " ".join(cmd) LOGGER.info("Running command '%s'...", cmd) kwargs.setdefault("check", True) - subprocess.run(shlex.split(cmd), **kwargs) # noqa: PLW1510, S603 + try: + subprocess.run(shlex.split(cmd), **kwargs) # noqa: PLW1510, S603 + except subprocess.CalledProcessError as e: + LOGGER.error(e.output) + LOGGER.error(str(e)) + sys.exit(e.returncode) LOGGER.info("Running command '%s'... done.", cmd) From a7d7ff1113055bd91c595e2ee3bd7ab38f80a1ce Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 12 Mar 2025 13:49:51 -0500 Subject: [PATCH 5/6] fix windows handling --- .evergreen/scripts/configure-env.sh | 42 ++++++++++++++--------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 7efabec0f3..fa37b8fb08 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -41,25 +41,6 @@ if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin PYMONGO_BIN_DIR=$(cygpath -m "$PYMONGO_BIN_DIR") fi -# If the toolchain is available, symlink binaries to the bin dir. -_bin_path="" -if [ "Windows_NT" == "${OS:-}" ]; then - _bin_path="/cygdrive/c/Python/Current/Scripts" -elif [ "$(uname -s)" != "Darwin" ]; then - _bin_path="/Library/Frameworks/Python.Framework/Versions/Current/bin" -else - _bin_path="/opt/python/Current/bin" -fi -if [ -d "${_bin_path}" ]; then - _suffix="" - if [ "Windows_NT" == "${OS:-}" ]; then - _suffix=".exe" - fi - ln -s ${_bin_path}/just${_suffix} $PYMONGO_BIN_DIR/just${_suffix} - ln -s ${_bin_path}/uv${_suffix} $PYMONGO_BIN_DIR/uv${_suffix} - ln -s ${_bin_path}/uvx${_suffix} $PYMONGO_BIN_DIR/uvx${_suffix} -fi - SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts" if [ -f "$SCRIPT_DIR/env.sh" ]; then @@ -97,8 +78,6 @@ EOT rm -rf $DRIVERS_TOOLS BRANCH=master ORG=mongodb-labs -BRANCH=PYTHON-5203 -ORG=blink1073 git clone --branch $BRANCH https://github.com/$ORG/drivers-evergreen-tools.git $DRIVERS_TOOLS cat < ${DRIVERS_TOOLS}/.env @@ -113,3 +92,24 @@ cat < expansion.yml DRIVERS_TOOLS: "$DRIVERS_TOOLS" PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" EOT + +# If the toolchain is available, symlink binaries to the bin dir. This has to be done +# after drivers-tools is cloned, since we might be using its binary dir. +_bin_path="" +if [ "Windows_NT" == "${OS:-}" ]; then + _bin_path="/cygdrive/c/Python/Current/Scripts" +elif [ "$(uname -s)" != "Darwin" ]; then + _bin_path="/Library/Frameworks/Python.Framework/Versions/Current/bin" +else + _bin_path="/opt/python/Current/bin" +fi +if [ -d "${_bin_path}" ]; then + _suffix="" + if [ "Windows_NT" == "${OS:-}" ]; then + _suffix=".exe" + fi + mkdir -p $PYMONGO_BIN_DIR + ln -s ${_bin_path}/just${_suffix} $PYMONGO_BIN_DIR/just${_suffix} + ln -s ${_bin_path}/uv${_suffix} $PYMONGO_BIN_DIR/uv${_suffix} + ln -s ${_bin_path}/uvx${_suffix} $PYMONGO_BIN_DIR/uvx${_suffix} +fi From 0c5a086647e9cd80be6828e7f5d02e99d2bfb345 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 12 Mar 2025 17:12:20 -0500 Subject: [PATCH 6/6] clean up handling of find_python3 --- .evergreen/utils.sh | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index e044b3d766..bb3ed8dabd 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -4,31 +4,29 @@ set -eu find_python3() { PYTHON="" - # Add a fallback system python3 if it is available and Python 3.9+. - if is_python_39 "$(command -v python3)"; then - PYTHON="$(command -v python3)" - fi # Find a suitable toolchain version, if available. if [ "$(uname -s)" = "Darwin" ]; then - # macos 11.00 - if [ -d "/Library/Frameworks/Python.Framework/Versions/3.10" ]; then - PYTHON="/Library/Frameworks/Python.Framework/Versions/3.10/bin/python3" - # macos 10.14 - elif [ -d "/Library/Frameworks/Python.Framework/Versions/3.9" ]; then - PYTHON="/Library/Frameworks/Python.Framework/Versions/3.9/bin/python3" - fi + PYTHON="/Library/Frameworks/Python.Framework/Versions/Current/bin/python3" elif [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin - PYTHON="C:/python/Python39/python.exe" + PYTHON="C:/python/Current/python.exe" else # Prefer our own toolchain, fall back to mongodb toolchain if it has Python 3.9+. - if [ -f "/opt/python/3.9/bin/python3" ]; then - PYTHON="/opt/python/3.9/bin/python3" + if [ -f "/opt/python/Current/bin/python3" ]; then + PYTHON="/opt/python/Current/bin/python3" + elif is_python_39 "$(command -v /opt/mongodbtoolchain/v5/bin/python3)"; then + PYTHON="/opt/mongodbtoolchain/v5/bin/python3" elif is_python_39 "$(command -v /opt/mongodbtoolchain/v4/bin/python3)"; then PYTHON="/opt/mongodbtoolchain/v4/bin/python3" elif is_python_39 "$(command -v /opt/mongodbtoolchain/v3/bin/python3)"; then PYTHON="/opt/mongodbtoolchain/v3/bin/python3" fi fi + # Add a fallback system python3 if it is available and Python 3.9+. + if [ -z "$PYTHON" ]; then + if is_python_39 "$(command -v python3)"; then + PYTHON="$(command -v python3)" + fi + fi if [ -z "$PYTHON" ]; then echo "Cannot test without python3.9+ installed!" exit 1