diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 8dc328aab3..a92c0b023a 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -14,31 +14,45 @@ fi PROJECT_DIRECTORY="$(pwd)" DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools" CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo} -UV_TOOL_DIR=$PROJECT_DIRECTORY/.local/uv/tools -UV_CACHE_DIR=$PROJECT_DIRECTORY/.local/uv/cache DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin" MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" -# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME. +# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME or +# have binaries shared across tasks, so use a TMPDIR. On non-CI hosts +# (spawn hosts, VMs such as GCP/Azure, and local dev), use the conventional +# ~/.local/bin which tools on the PATH (or the shell rc) can find. if [ "${CI:-}" == "true" ]; then - PYMONGO_BIN_DIR=${DRIVERS_TOOLS_BINARIES:-} -# We want to use a path that's already on PATH on spawn hosts. + PYMONGO_BIN_DIR="${TMPDIR:-/tmp}"/pymongo_bin else - PYMONGO_BIN_DIR=$HOME/cli_bin + PYMONGO_BIN_DIR=$HOME/.local/bin fi -PATH_EXT="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PYMONGO_BIN_DIR:\$PATH" +# Add the latest MongoDB toolchain bin dir to PATH if it exists, so that hosts +# with an old system Python (e.g. RHEL8's 3.6) still get a modern interpreter +# for tool installs like `uv tool install rust-just`. It goes after +# PYMONGO_BIN_DIR so the pinned uv (installed there by setup-uv.py) takes +# precedence over the toolchain's uv. +if [ "Windows_NT" = "${OS:-}" ]; then + _toolchain_bin="/cygdrive/c/Python/Current/Scripts" +elif [ "$(uname -s)" == "Darwin" ]; then + _toolchain_bin="/Library/Frameworks/Python.Framework/Versions/Current/bin" +else + _toolchain_bin="/opt/python/Current/bin" +fi +if [ -d "$_toolchain_bin" ]; then + PATH_EXT="$MONGODB_BINARIES:$PYMONGO_BIN_DIR:$_toolchain_bin:$DRIVERS_TOOLS_BINARIES:\$PATH" +else + PATH_EXT="$MONGODB_BINARIES:$PYMONGO_BIN_DIR:$DRIVERS_TOOLS_BINARIES:\$PATH" +fi # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY) CARGO_HOME=$(cygpath -m $CARGO_HOME) - UV_TOOL_DIR=$(cygpath -m "$UV_TOOL_DIR") - UV_CACHE_DIR=$(cygpath -m "$UV_CACHE_DIR") DRIVERS_TOOLS_BINARIES=$(cygpath -m "$DRIVERS_TOOLS_BINARIES") MONGODB_BINARIES=$(cygpath -m "$MONGODB_BINARIES") - PYMONGO_BIN_DIR=$(cygpath -m "$PYMONGO_BIN_DIR") + PYMONGO_BIN_DIR=$(cygpath -u "$PYMONGO_BIN_DIR") fi SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts" @@ -62,9 +76,6 @@ export DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS_BINARIES" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" export CARGO_HOME="$CARGO_HOME" -export UV_TOOL_DIR="$UV_TOOL_DIR" -export UV_CACHE_DIR="$UV_CACHE_DIR" -export UV_TOOL_BIN_DIR="$DRIVERS_TOOLS_BINARIES" export PYMONGO_BIN_DIR="$PYMONGO_BIN_DIR" export PATH="$PATH_EXT" # shellcheck disable=SC2154 @@ -90,25 +101,3 @@ cat < expansion.yml DRIVERS_TOOLS: "$DRIVERS_TOOLS" PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" EOT - -# If the toolchain is available, symlink binaries to the bin dir. This has to be done -# after drivers-tools is cloned, since we might be using its binary dir. -_bin_path="" -if [ "Windows_NT" == "${OS:-}" ]; then - _bin_path="/cygdrive/c/Python/Current/Scripts" -elif [ "$(uname -s)" == "Darwin" ]; then - _bin_path="/Library/Frameworks/Python.Framework/Versions/Current/bin" -else - _bin_path="/opt/python/Current/bin" -fi -if [ -d "${_bin_path}" ]; then - _suffix="" - if [ "Windows_NT" == "${OS:-}" ]; then - _suffix=".exe" - fi - echo "Symlinking binaries from toolchain" - mkdir -p $PYMONGO_BIN_DIR - ln -s ${_bin_path}/just${_suffix} $PYMONGO_BIN_DIR/just${_suffix} - ln -s ${_bin_path}/uv${_suffix} $PYMONGO_BIN_DIR/uv${_suffix} - ln -s ${_bin_path}/uvx${_suffix} $PYMONGO_BIN_DIR/uvx${_suffix} -fi diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh index 7f943d7d00..b701b914a2 100755 --- a/.evergreen/scripts/install-dependencies.sh +++ b/.evergreen/scripts/install-dependencies.sh @@ -1,6 +1,6 @@ #!/bin/bash # Install the necessary dependencies. -set -eu +set -euo pipefail HERE=$(dirname ${BASH_SOURCE:-$0}) HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" @@ -11,27 +11,71 @@ if [ -f $HERE/env.sh ]; then . $HERE/env.sh fi -# Set up the default bin directory. -if [ -z "${PYMONGO_BIN_DIR:-}" ]; then - PYMONGO_BIN_DIR="$HOME/.local/bin" +# PYMONGO_BIN_DIR is set by setup-system.sh/env.sh (or setup-dev-env.sh); default +# it for robustness. UV_TOOL_BIN_DIR is uv's name for the same dir (setup-uv.py +# reads both). UV_TOOL_DIR is left to ensure_uv.sh. +export PYMONGO_BIN_DIR="${PYMONGO_BIN_DIR:-$HOME/.local/bin}" +export UV_TOOL_BIN_DIR="${UV_TOOL_BIN_DIR:-$PYMONGO_BIN_DIR}" +# uv is a native Windows binary: give it a Windows path on cygwin. +if [ "Windows_NT" = "${OS:-}" ]; then + _uv_tool_bin="$(cygpath -m "$PYMONGO_BIN_DIR")" + export UV_TOOL_BIN_DIR="$_uv_tool_bin" fi -# Ensure uv is installed. -if ! command -v uv &>/dev/null; then - _BIN_DIR=$PYMONGO_BIN_DIR - mkdir -p ${_BIN_DIR} - echo "Installing uv..." - curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$_BIN_DIR" INSTALLER_NO_MODIFY_PATH=1 sh - if [ "Windows_NT" = "${OS:-}" ]; then - chmod +x "$(cygpath -u $_BIN_DIR)/uv.exe" +# Ensure the bin dir is on PATH: hosts without env.sh (e.g. auth-aws-ecs) never +# export it, so a fresh pinned install there would be invisible to the probe +# below and to later steps like `uv tool install` and `uv sync`. +case ":$PATH:" in + *":$PYMONGO_BIN_DIR:"*) ;; + *) export PATH="$PYMONGO_BIN_DIR:$PATH" ;; +esac + +# Compute the setup script path once (native Windows path on cygwin). +_uv_setup_script="$HERE/setup-uv.py" +if [ "Windows_NT" = "${OS:-}" ]; then + _uv_setup_script="$(cygpath -m "$_uv_setup_script")" +fi + +# Skip setup when the pinned uv is already installed: `setup-uv.py --check` +# compares the uv on PATH with the required-version pin, without side effects. +# A full `uv sync` here would download Python and install/build dependencies +# just to make this decision, and would run before setup-uv-python.sh sets +# UV_PYTHON, creating the environment twice on local dev. +# +# On CI we also require UV_CACHE_DIR to be set: ensure_uv.sh scopes uv's cache +# to a task-local dir, so an unset UV_CACHE_DIR means the uv setup has not run +# yet in this task and we must do the setup phase. +_need_setup=1 +if [ "${CI:-}" != "true" ] || [ -n "${UV_CACHE_DIR:-}" ]; then + if python3 "$_uv_setup_script" --check >/dev/null 2>&1; then + echo "uv is already set up; skipping uv setup." + _need_setup=0 + fi +fi + +# Set up uv if needed. +if [ "$_need_setup" = "1" ]; then + # ensure-uv.sh (drivers-evergreen-tools) finds or installs uv and scopes its env. + if [ -n "${DRIVERS_TOOLS:-}" ] && [ -f "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ]; then + . "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" + ensure_uv || exit 1 + fi + + # Do the uv setup (bin dir, pinning, env.sh). Uses the toolchain python3 + # (added to PATH by configure-env.sh) so no project .venv is created here, + # and no required-version check is triggered. + python3 "$_uv_setup_script" + + # Re-source env.sh so the values setup-uv.py wrote are available. + if [ -f $HERE/env.sh ]; then + . $HERE/env.sh fi - export PATH="$PYMONGO_BIN_DIR:$PATH" - echo "Installing uv... done." fi -# Ensure just is installed. -if ! command -v just &>/dev/null; then - uv tool install rust-just +# Make just available. It has no version constraint, so if it is already on PATH +# there is nothing to do; otherwise install it into the bin dir via uv. +if ! command -v just >/dev/null 2>&1; then + uv tool install --no-config rust-just fi popd > /dev/null diff --git a/.evergreen/scripts/setup-dev-env.sh b/.evergreen/scripts/setup-dev-env.sh index e58d6210fe..5166c4b523 100755 --- a/.evergreen/scripts/setup-dev-env.sh +++ b/.evergreen/scripts/setup-dev-env.sh @@ -16,22 +16,45 @@ if [ -f $HERE/test-env.sh ]; then . $HERE/test-env.sh fi -# Handle the value for UV_PYTHON. -. $HERE/setup-uv-python.sh +# The bin dir for the pinned uv/just. setup-system.sh sets it on evergreen hosts; +# default it here so local dev (without setup-system.sh) also has a usable value. +export PYMONGO_BIN_DIR="${PYMONGO_BIN_DIR:-$HOME/.local/bin}" + +# install-dependencies.sh runs as a child process, so its PATH changes do not +# propagate back here: ensure the bin dir is on this process's PATH too, so a +# fresh install (first run, bin dir not on PATH yet) is visible to the +# `uv sync` and pre-commit setup below. +case ":$PATH:" in + *":$PYMONGO_BIN_DIR:"*) ;; + *) export PATH="$PYMONGO_BIN_DIR:$PATH" ;; +esac + +# Make sure a login shell can find the bin dir by adding it to the rc file, so +# local dev (which may never run setup-system.sh) still has it on PATH. env.sh's +# PATH does not persist past this session. Select the rc file from $SHELL (not +# by which rc file happens to exist) so the user's actual shell is updated, and +# create it if it does not exist yet. +if [ "${CI:-}" != "true" ] && [ "${GITHUB_ACTIONS:-}" != "true" ]; then + case "${SHELL:-}" in + */zsh) _rc="$HOME/.zshrc" ;; + *) _rc="$HOME/.bashrc" ;; + esac + touch "$_rc" + grep -qF 'export PATH="'"$PYMONGO_BIN_DIR"':$PATH"' "$_rc" 2>/dev/null || \ + printf 'export PATH="%s:$PATH"\n' "$PYMONGO_BIN_DIR" >> "$_rc" +fi # Ensure dependencies are installed. bash $HERE/install-dependencies.sh -# Re-source env.sh: install-dependencies.sh may have appended to it, e.g. when it -# had to install Python on an image that lacks a toolchain. +# Re-source env.sh in case a dependency install updated it, e.g. on a host +# without a toolchain where uv was installed into a shared bin dir. if [ -f $HERE/env.sh ]; then . $HERE/env.sh fi -# Add the default install path to the path if needed. -if [ -z "${PYMONGO_BIN_DIR:-}" ]; then - export PATH="$PATH:$HOME/.local/bin" -fi +# Handle the value for UV_PYTHON. +. $HERE/setup-uv-python.sh # Only run the next part if not running on CI. if [ -z "${CI:-}" ]; then diff --git a/.evergreen/scripts/setup-uv.py b/.evergreen/scripts/setup-uv.py new file mode 100755 index 0000000000..dc7b0964f4 --- /dev/null +++ b/.evergreen/scripts/setup-uv.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Bootstrap the pinned uv/just for the test environment. + +install-dependencies.sh probes with `setup-uv.py --check`, sources ensure-uv.sh +(which finds or installs uv), then runs this script for the rest. Only the +standard library is used. +""" + +from __future__ import annotations + +import os +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent.parent +ENV_SH = HERE / "env.sh" +ASTRAL_INSTALL_URL = "https://astral.sh/uv/install.sh" + + +def required_uv_pin() -> str: + """Return [tool.uv] required-version, e.g. '==0.12.12' ('' if absent).""" + pattern = re.compile(r"required-version\s*=\s*['\"]?([^'\"\s]+)['\"]?") + in_uv = False + for line in (ROOT / "pyproject.toml").read_text().splitlines(): + stripped = line.strip() + if stripped.startswith("["): + in_uv = stripped == "[tool.uv]" + continue + if in_uv: + match = pattern.search(stripped) + if match: + return match.group(1) + return "" + + +def _add_path(dir_: str) -> None: + os.environ["PATH"] = dir_ + os.pathsep + os.environ.get("PATH", "") + + +def _install_uv_astral() -> None: + """Install uv from astral when ensure-uv.sh was not available. + + UV_TOOL_BIN_DIR is set (in native form on Windows) by install-dependencies.sh. + """ + print("uv not found; installing the latest uv from astral...") + env = { + **os.environ, + "UV_INSTALL_DIR": os.environ["UV_TOOL_BIN_DIR"], + "INSTALLER_NO_MODIFY_PATH": "1", + } + curl = shutil.which("curl") + sh = shutil.which("sh") + proc = subprocess.run( # noqa: S603 + [curl, "-LsSf", ASTRAL_INSTALL_URL], capture_output=True, env=env, check=True + ) + subprocess.run([sh], input=proc.stdout, env=env, check=True) # noqa: S603 + _add_path(os.environ["UV_TOOL_BIN_DIR"]) + + +def _pin_uv(uv_pin: str) -> None: + """Install the pinned uv via uv tool install --force. + + UV_TOOL_BIN_DIR and UV_TOOL_DIR are inherited from the environment (set by + install-dependencies.sh / ensure-uv.sh, in native form on Windows). uv tool + install writes the binary into UV_TOOL_BIN_DIR and the tool venv into + UV_TOOL_DIR; --force lets it overwrite an existing install. + + Windows will not overwrite a running executable, so when the current uv is + already the target bin dir, run the install from a copy of the binary. + """ + uv_path = shutil.which("uv") + tool = uv_path + tmp_uv = None + if os.name == "nt": + tmp_uv = Path(tempfile.mkdtemp()) / Path(uv_path).name + shutil.copy2(uv_path, tmp_uv) + tool = str(tmp_uv) + try: + subprocess.run( # noqa: S603 + [ + tool, + "tool", + "install", + "--no-config", + "-q", + "--force", + "--from", + f"uv{uv_pin}", + "uv", + ], + check=True, + ) + finally: + if tmp_uv: + shutil.rmtree(tmp_uv.parent, ignore_errors=True) + + +# The UV_* env vars this bootstrap (or its caller, ensure-uv.sh) manages and +# may persist into env.sh. Deliberately narrow: other UV_* vars (e.g. +# UV_PUBLISH_TOKEN, UV_INDEX_..._PASSWORD) may hold credentials and must +# never be written out. +PERSISTED_UV_VARS = ("UV_CACHE_DIR", "UV_PYTHON_INSTALL_DIR", "UV_TOOL_BIN_DIR", "UV_TOOL_DIR") + + +def _write_env() -> None: + """Write the bootstrap's UV_* env vars into env.sh, replacing existing UV_* entries.""" + values = {k: v for k, v in os.environ.items() if k in PERSISTED_UV_VARS} + if not values: + return + existing = ENV_SH.read_text() if ENV_SH.exists() else "" + keep = [] + for line in existing.splitlines(): + stripped = line.strip() + if stripped.startswith("export "): + var, _, _ = stripped[len("export ") :].partition("=") + if var.startswith("UV_"): + continue + keep.append(line) + keep.append("") + # shlex.quote emits shell-safe values (single-quoted, escaping embedded + # quotes) so a value cannot break out of or inject into the assignment. + keep.extend(f"export {name}={shlex.quote(value)}" for name, value in sorted(values.items())) + # Write LF bytes directly: on Windows text mode translates \n to \r\n, which + # breaks bash sourcing env.sh, and `newline=` isn't available on all Pythons. + ENV_SH.write_bytes(("\n".join(keep) + "\n").encode()) + + +def _pinned_uv_installed() -> bool: + """Return True when the uv on PATH already matches the required-version pin.""" + pin = required_uv_pin() + if not pin.startswith("=="): + # Only exact pins are checked here; anything else falls back to setup. + return False + uv_path = shutil.which("uv") + if uv_path is None: + return False + proc = subprocess.run( # noqa: S603 + [uv_path, "--version"], capture_output=True, check=True + ) + # Output looks like "uv 0.12.12 (aarch64-unknown-linux-gnu)". + return proc.stdout.decode().split()[1] == pin[len("==") :] + + +def main() -> int: + # Side-effect-free probe for install-dependencies.sh: a full `uv sync` + # there would download Python and install dependencies just to decide + # whether setup is needed. + if "--check" in sys.argv[1:]: + return 0 if _pinned_uv_installed() else 1 + + bin_dir = os.environ["UV_TOOL_BIN_DIR"] + Path(bin_dir).mkdir(parents=True, exist_ok=True) + _add_path(bin_dir) + + # Bootstrap a uv from astral if ensure-uv.sh didn't run and uv isn't on PATH. + if shutil.which("uv") is None: + _install_uv_astral() + + uv_pin = required_uv_pin() + if uv_pin: + _pin_uv(uv_pin) + + _write_env() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.gitignore b/.gitignore index 6c4a512018..8546aeabb7 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ mongocryptd.pid .nova/ .temp/ venv/ +.local/ secrets-export.sh libmongocrypt.tar.gz libmongocrypt/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 146a0adb3c..20b7210736 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -199,6 +199,10 @@ the pages will re-render and the browser will automatically refresh. - Run `just install` to set a local virtual environment, or you can manually create a virtual environment and run `pytest` directly. If you want to use a specific version of Python, set `UV_PYTHON` before running `just install`. + + `just install` installs the pinned version of `uv` (from `[tool.uv] required-version`) into `$HOME/.local/bin`, + and adds that directory to your shell rc file when missing, so the pinned `uv` takes effect in new shells. If a + project `uv` command (e.g. `just test`) runs with a different `uv` version, `uv` fails fast and tells you how to update. - Ensure you have started the appropriate Mongo Server(s). You can run `just run-server` with optional args to set up the server. All given options will be passed to [`run-mongodb.sh`](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-mongodb.sh). Run `$DRIVERS_TOOLS/.evergreen/run-mongodb.sh start -h` @@ -408,6 +412,10 @@ tasks are host-agnostic. supported version of Python and use that. This ensures a consistent behavior across host types that do not have the Python toolchain (e.g. Azure VMs), by having a known version of Python with the build headers (`Python.h`) needed to build the C extensions. + - The uv binary version is pinned once in `[tool.uv] required-version` in `pyproject.toml`. + `.evergreen/scripts/install-dependencies.sh` installs it with `uv tool install`, uv enforces it locally, and + `astral-sh/setup-uv` reads it on GitHub. Bump it manually when a newer uv is needed. If uv cannot find the + requested Python, it installs it; if that fails, the task fails. - Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`. - Make sure to add instructions for running the test suite to `CONTRIBUTING.md`. diff --git a/pyproject.toml b/pyproject.toml index 2adddcd29b..d821fa8673 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,11 @@ Source = "https://github.com/mongodb/mongo-python-driver" Tracker = "https://jira.mongodb.org/projects/PYTHON/issues" [tool.uv] +# Pin the uv binary version across local dev, GitHub Actions, and Evergreen. +# uv enforces this locally, astral-sh/setup-uv reads it on GitHub, and +# install-dependencies.sh installs it in Evergreen. Bump manually when a newer uv +# is needed. +required-version = "==0.12.12" # boto3 dropped Python 3.9 support in 1.43, so the universal lock forks at # 3.10. Without a floor the pre-3.10 fork back-solves to boto3 1.7.84 (2018), # whose vendored six and invalid escape sequences break test collection. No