diff --git a/.github/workflows/check_changelogs.yml b/.github/workflows/check_changelogs.yml index 25034b868d..420ae56492 100644 --- a/.github/workflows/check_changelogs.yml +++ b/.github/workflows/check_changelogs.yml @@ -25,7 +25,7 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - name: Check zarr-python changelog entries - run: uv run --no-sync python ci/check_changelog_entries.py + run: uvx --from rust-just just check-changelogs - name: Check zarr-metadata changelog entries - run: uv run --no-sync python ci/check_changelog_entries.py packages/zarr-metadata/changes + run: uvx --from rust-just just check-changelogs packages/zarr-metadata/changes diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index a77e99bc28..a97ef1ad75 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -27,12 +27,17 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - name: Install Hatch - uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: - version: '1.16.5' + enable-cache: true + python-version: '3.12' + # Pre-build the locked env so the measured `run` below is just pytest, not a + # cold uv sync — keeps the walltime sample clean. + - name: Sync locked benchmark env + run: uv sync --locked --no-default-groups --group test -p 3.12 - name: Run the benchmarks uses: CodSpeedHQ/action@63f3e98b61959fe67f146a3ff022e4136fe9bb9c # v4.17.6 with: mode: walltime - run: hatch run test.py3.12-minimal:pytest tests/benchmarks --codspeed + run: uv run --no-sync -p 3.12 pytest tests/benchmarks --codspeed diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 59eca8d9b9..d1da8c4afe 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,13 +23,11 @@ jobs: with: persist-credentials: false - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - - run: uv sync --group docs - # --strict turns warnings into errors, so a docs code block that fails to execute - # at build time (e.g. a non-exec python fence disrupting a later exec="true" block) - # fails CI instead of merging as a silent warning. - - run: uv run mkdocs build --strict - env: - DISABLE_MKDOCS_2_WARNING: "true" - NO_MKDOCS_2_WARNING: "true" - - run: uv run python ci/check_unlinked_types.py + with: + python-version: '3.12' + # `just docs-build` runs `mkdocs build --strict` (warnings are errors, so a + # docs code block that fails to execute at build time fails CI) against the + # locked docs env. check_unlinked_types runs in that same synced env. + - run: uvx --from rust-just just docs-build + - run: uvx --from rust-just just check-doc-links continue-on-error: true diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index 60c871cb15..6a9c408214 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -12,8 +12,6 @@ on: env: LD_LIBRARY_PATH: /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64 - # Use the uv from astral-sh/setup-uv instead of hatch's bundled (pyapp) uv. - HATCH_ENV_TYPE_VIRTUAL_UV_PATH: uv permissions: contents: read @@ -56,28 +54,13 @@ jobs: echo $PATH echo $LD_LIBRARY_PATH nvcc -V - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - - name: Install Hatch - uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc with: - version: '1.16.5' - - name: Set Up Hatch Env - env: - HATCH_ENV: gputest.py${{ matrix.python-version }} - run: | - hatch env create "$HATCH_ENV" - hatch env run -e "$HATCH_ENV" list-env + enable-cache: true + python-version: ${{ matrix.python-version }} - name: Run Tests - env: - HATCH_ENV: gputest.py${{ matrix.python-version }} - run: | - hatch env run --env "$HATCH_ENV" run-coverage + run: uvx --from rust-just just gpu - name: Upload coverage uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index b1262cee06..b72288fc35 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -18,8 +18,6 @@ concurrency: env: FORCE_COLOR: 3 - # Use the uv from astral-sh/setup-uv instead of hatch's bundled (pyapp) uv. - HATCH_ENV_TYPE_VIRTUAL_UV_PATH: uv jobs: @@ -51,23 +49,11 @@ jobs: else echo "HYPOTHESIS_PROFILE=ci" >> $GITHUB_ENV fi - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - - name: Install Hatch - uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc with: - version: '1.16.5' - - name: Set Up Hatch Env - env: - HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} - run: | - hatch env create "$HATCH_ENV" - hatch env run -e "$HATCH_ENV" list-env + enable-cache: true + python-version: ${{ matrix.python-version }} # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache - name: Restore cached hypothesis directory id: restore-hypothesis-cache @@ -81,11 +67,9 @@ jobs: - name: Run slow Hypothesis tests if: success() id: status - env: - HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} run: | echo "Using Hypothesis profile: $HYPOTHESIS_PROFILE" - hatch env run --env "$HATCH_ENV" run-hypothesis + uvx --from rust-just just hypothesis # explicitly save the cache so it gets updated, also do this even if it fails. - name: Save cached hypothesis directory diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d60c9f59b4..253f176c99 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,4 +30,9 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true + # Fail fast if uv.lock is stale vs pyproject.toml. Every test job installs + # with `uv sync --locked`, so a drifted lock would otherwise only surface + # as a confusing failure there. + - name: Check uv.lock is up to date + run: uv lock --check - uses: j178/prek-action@bdca6f102f98e2b4c7029491a53dfd366469e33d # v2.0.4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a75974f6c9..46039e46d7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,11 +13,6 @@ on: permissions: contents: read -env: - # Use the uv from astral-sh/setup-uv; without an explicit path hatch - # bootstraps its own (pyapp) uv, which fails on non-3.12 runners. - HATCH_ENV_TYPE_VIRTUAL_UV_PATH: uv - concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -60,27 +55,21 @@ jobs: with: fetch-depth: 0 # grab all branches and tags persist-credentials: false - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - - name: Install Hatch - run: python -m pip install hatch==1.16.5 - - name: Set Up Hatch Env - env: - HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} - run: | - hatch env create "$HATCH_ENV" - hatch env run -e "$HATCH_ENV" list-env + with: + enable-cache: true + python-version: ${{ matrix.python-version }} + # The `just` recipe (see Justfile) runs `uv sync --locked`, so one universal + # uv.lock serves every os/python cell — and it fails if the lock is stale vs + # pyproject.toml (a cross-platform drift gate). setup-uv's python-version + # sets UV_PYTHON so uv picks the right interpreter. CI and local dev invoke + # the identical recipe; `uvx --from rust-just` provides `just` without a + # separate install. - name: Run Tests env: HYPOTHESIS_PROFILE: ci - HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} - run: | - hatch env run --env "$HATCH_ENV" run-coverage + run: uvx --from rust-just just test-${{ matrix.dependency-set }} - name: Upload coverage if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 @@ -109,26 +98,16 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - - name: Install Hatch - run: python -m pip install hatch==1.16.5 - - name: Set Up Hatch Env - env: - HATCH_ENV: ${{ matrix.dependency-set }} - run: | - hatch env create "$HATCH_ENV" - hatch env run -e "$HATCH_ENV" list-env + with: + enable-cache: true + python-version: ${{ matrix.python-version }} + # min_deps and upstream are intentionally unlocked (floor constraints from + # ci/min-deps-constraints.txt; nightly + git mains overlay). The recipe names + # match the matrix values — see the Justfile. - name: Run Tests - env: - HATCH_ENV: ${{ matrix.dependency-set }} - run: | - hatch env run --env "$HATCH_ENV" run-coverage + run: uvx --from rust-just just ${{ matrix.dependency-set }} - name: Upload coverage uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: @@ -144,21 +123,13 @@ jobs: with: fetch-depth: 0 # required for hatch version discovery, which is needed for numcodecs.zarr3 persist-credentials: false - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: '3.13' - cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - - name: Install Hatch - run: python -m pip install hatch==1.16.5 - - name: Set Up Hatch Env - run: | - hatch run doctest:pip list - - name: Run Tests - run: | - hatch run doctest:test + with: + enable-cache: true + python-version: '3.13' + - name: Run doctests + run: uvx --from rust-just just doctest benchmarks: name: Benchmark smoke test @@ -168,18 +139,13 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: '3.13' - cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - - name: Install Hatch - run: python -m pip install hatch==1.16.5 + with: + enable-cache: true + python-version: '3.13' - name: Run Benchmarks - run: | - hatch env run --env "test.py3.13-minimal" run-benchmark + run: uvx --from rust-just just benchmark test-complete: name: Test complete diff --git a/.gitignore b/.gitignore index 3284865d6c..2b61e1efda 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ .Python env/ .venv/ +.venv-min-deps/ build/ develop-eggs/ dist/ diff --git a/Justfile b/Justfile new file mode 100644 index 0000000000..ea08508999 --- /dev/null +++ b/Justfile @@ -0,0 +1,154 @@ +# zarr-python developer task runner (https://github.com/casey/just). +# +# `just` is a thin verb-runner over uv; uv owns the environments. Each recipe is +# the single source of truth for a dev/CI task — CI calls the same recipes (via +# `uvx --from rust-just just `), so local and CI behavior cannot drift. +# +# Install just: uv tool install rust-just (or `brew install just`, `cargo install just`) +# List recipes: just (or `just --list`) +# +# The matrix lives in GitHub Actions; pass the Python version via UV_PYTHON +# (setup-uv sets it from `python-version`). Locally, override per call, e.g. +# UV_PYTHON=3.13 just test-optional + +# Extras + groups that make up the "optional" (full integration) test environment. +optional_deps := "--extra remote --extra optional --extra cli --extra cast-value-rs --group remote-tests" + +[private] +default: + @just --list + +[doc("Run the unit tests with the minimal dependency set")] +test-minimal *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks \ + --junitxml=junit.xml -o junit_family=legacy {{args}} + uv run --no-sync coverage xml + +[doc("Run the unit tests with the full (optional) integration dependency set")] +test-optional *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test {{optional_deps}} + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks \ + --junitxml=junit.xml -o junit_family=legacy {{args}} + uv run --no-sync coverage xml + +[doc("Generate an HTML coverage report (optional deps); open htmlcov/index.html")] +coverage-html *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test {{optional_deps}} + uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks {{args}} + uv run --no-sync coverage html + +[doc("Run the slow Hypothesis property tests")] +hypothesis *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test {{optional_deps}} + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest -nauto --run-slow-hypothesis \ + tests/test_properties.py tests/test_store/test_stateful* {{args}} + uv run --no-sync coverage xml + +[doc("Validate executable code blocks in the docs (tests/test_docs.py)")] +doctest *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --extra remote --group remote-tests + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync --with pytest-examples pytest tests/test_docs.py -v {{args}} + +[doc("Run the benchmark suite (minimal deps)")] +benchmark *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test + uv run --no-sync pytest --benchmark-enable tests/benchmarks {{args}} + +[doc("Run the tests against the lowest supported direct dependency versions")] +min_deps *args: + #!/usr/bin/env bash + set -euo pipefail + # Build the env imperatively with `uv pip` (does not read/write uv.lock) so + # the floors in ci/min-deps-constraints.txt apply only here. Runtime deps are + # pinned to their floors; everything else (e.g. flask/werkzeug) resolves to + # its latest compatible release. Isolated venv so the dev .venv is untouched. + export UV_PROJECT_ENVIRONMENT="${UV_PROJECT_ENVIRONMENT:-.venv-min-deps}" + export VIRTUAL_ENV="$UV_PROJECT_ENVIRONMENT" + uv venv --clear "$UV_PROJECT_ENVIRONMENT" + uv pip install --editable '.[remote,optional]' --group test --group remote-tests \ + --constraint ci/min-deps-constraints.txt + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks \ + --junitxml=junit.xml -o junit_family=legacy {{args}} + uv run --no-sync coverage xml + +[doc("Run the tests against bleeding-edge (nightly + git main) dependencies")] +upstream *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --no-default-groups --group test --group remote-tests --extra remote + uv pip install --prerelease=allow \ + --index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/ \ + --extra-index-url https://pypi.org/simple/ \ + numpy \ + "packaging @ git+https://github.com/pypa/packaging" \ + "numcodecs @ git+https://github.com/zarr-developers/numcodecs" \ + "s3fs @ git+https://github.com/fsspec/s3fs" \ + "universal_pathlib @ git+https://github.com/fsspec/universal_pathlib" \ + "typing_extensions @ git+https://github.com/python/typing_extensions" \ + "donfig @ git+https://github.com/pytroll/donfig" \ + "obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore" + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks \ + --junitxml=junit.xml -o junit_family=legacy {{args}} + uv run --no-sync coverage xml + +[doc("Run the GPU tests (requires CUDA + a GPU); `pytest -m gpu`")] +gpu *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test --extra gpu --extra optional + uv pip install pytest-examples + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest -m gpu --ignore tests/benchmarks \ + --junitxml=junit.xml -o junit_family=legacy {{args}} + uv run --no-sync coverage xml + +[doc("Build the documentation (strict: warnings are errors)")] +docs-build *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --extra remote --group docs + DISABLE_MKDOCS_2_WARNING=true NO_MKDOCS_2_WARNING=true \ + uv run --no-sync mkdocs build --strict {{args}} + +[doc("Serve the documentation locally with live reload at http://0.0.0.0:8000/")] +docs-serve *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --extra remote --group docs + DISABLE_MKDOCS_2_WARNING=true NO_MKDOCS_2_WARNING=true \ + uv run --no-sync mkdocs serve --watch src {{args}} + +[doc("Run all pre-commit hooks (ruff, codespell, mypy, repo-review, ...)")] +lint *args: + prek run --all-files {{args}} + +[doc("Check that uv.lock is in sync with pyproject.toml")] +lock-check: + uv lock --check + +[doc("Check changelog entry filenames (pass a directory to check, default: changes/)")] +check-changelogs *dir: + uv run --no-sync python ci/check_changelog_entries.py {{dir}} + +[doc("Report unlinked types in the built docs (run `just docs-build` first)")] +check-doc-links *args: + uv run --no-sync python ci/check_unlinked_types.py {{args}} diff --git a/changes/4096.misc.md b/changes/4096.misc.md new file mode 100644 index 0000000000..f2c30699a7 --- /dev/null +++ b/changes/4096.misc.md @@ -0,0 +1 @@ +Replaced hatch with uv and [just](https://github.com/casey/just) for development environments and CI task management. Contributors create the development environment with `uv sync --locked` and run tasks via `just` recipes defined in the `Justfile` (e.g. `just test-optional`, `just docs-build`; run `just` to list them). The recipes are thin wrappers over uv and are shared by CI, so local and CI behavior cannot drift. The `tool.hatch.envs.*` environments were removed and a committed `uv.lock` now pins the full transitive dependency closure for reproducible, cross-platform CI. hatchling remains the build backend, so package builds and version derivation are unchanged. diff --git a/ci/min-deps-constraints.txt b/ci/min-deps-constraints.txt new file mode 100644 index 0000000000..3234e04c47 --- /dev/null +++ b/ci/min-deps-constraints.txt @@ -0,0 +1,32 @@ +# Lowest supported dependency versions (the SPEC 0 floor), exercised by the +# `min_deps` CI job (`just min_deps`). Each pin is the EXACT lowest version that +# satisfies the matching `>=` bound in pyproject.toml — so the job verifies the +# precise floor we advertise (not merely "some version in the floor's minor +# series"). Keep the two in sync. SPEC 0 drop schedule: +# https://scientific-python.org/specs/spec-0000/ +# +# These pin the *runtime* dependencies; transitive deps (e.g. flask, werkzeug) +# are left to resolve to their latest compatible release. +# +# Why a hand-pinned list, and not uv's `--resolution lowest-direct` (which would +# derive the floors automatically from the `>=` bounds)? +# * It tests the EXACT floor we advertise. lowest-direct resolves to the lowest +# *mutually consistent* set, which can sit above a declared floor and thus +# silently skip a broken minimum (it resolved packaging 23.0, not 22.0). +# * lowest-direct drives under-constrained *transitive* deps to their absolute +# floor too: moto only says `flask!=2.2.0,!=2.2.1`, so flask floored to 2.0.3 +# (which imports `url_quote`, removed in werkzeug 2.3+) -> ImportError. +# +# Applied with `uv pip install --constraint`, which does not touch uv.lock. The +# same pins in a [dependency-groups] entry would leak into the single universal +# lock and drag the locked envs down too. +packaging==22.0 +numpy==2.0.0 +numcodecs==0.14.1 +google-crc32c==1.5.0 +typing_extensions==4.14.0 +donfig==0.8.0 +fsspec==2023.10.0 +s3fs==2023.10.0 +obstore==0.5.1 +universal_pathlib==0.2.0 diff --git a/docs/contributing.md b/docs/contributing.md index 750f7c7a65..922e74a0d6 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -80,17 +80,25 @@ git remote add upstream git@github.com:zarr-developers/zarr-python.git ### Creating a development environment -To work with the Zarr source code, it is recommended to use [hatch](https://hatch.pypa.io/latest/index.html) to create and manage development environments. Hatch will automatically install all Zarr dependencies using the same versions as are used by the core developers and continuous integration services. Assuming you have a Python 3 interpreter already installed, and you have cloned the Zarr source code and your current working directory is the root of the repository, you can do something like the following: +To work with the Zarr source code, we recommend [uv](https://docs.astral.sh/uv/) to create and manage development environments. uv installs all Zarr dependencies pinned to the same versions used by continuous integration, via the committed `uv.lock`. Assuming you have [installed uv](https://docs.astral.sh/uv/getting-started/installation/), cloned the Zarr source code, and your current working directory is the root of the repository, run: ```bash -pip install hatch -hatch env show # list all available environments +uv sync --locked # create .venv/ with Zarr and all development dependencies ``` -To verify that your development environment is working, you can run the unit tests for one of the test environments, e.g.: +This creates a virtual environment in `.venv/`. Prefix commands with `uv run` to run them inside it, or activate it directly with `source .venv/bin/activate`. To use a specific Python version, pass `-p`, e.g. `uv sync --locked -p 3.13`. + +Common developer tasks (running the matrix test environments, building docs, etc.) are defined as [`just`](https://github.com/casey/just) recipes in the `Justfile`. Install `just` and list the available recipes with: + +```bash +uv tool install rust-just # or: brew install just +just # list all recipes +``` + +To verify that your development environment is working, you can run the unit tests: ```bash -hatch env run --env test.py3.12-optional run +uv run pytest ``` ### Creating a branch @@ -125,16 +133,40 @@ Again, any conflicts need to be resolved before submitting a pull request. ### Running the test suite -Zarr includes a suite of unit tests. The simplest way to run the unit tests is to activate your development environment (see [creating a development environment](#creating-a-development-environment) above) and invoke: +Zarr includes a suite of unit tests. The simplest way to run them is within your development environment (see [creating a development environment](#creating-a-development-environment) above): + +```bash +uv run pytest +``` + +To reproduce a CI test environment exactly — the same dependency set CI uses, with coverage — run the matching `just` recipe. These sync the locked environment and run the test suite the way CI does: ```bash -hatch env run --env test.py3.12-optional run +just test-minimal # the minimal dependency set +just test-optional # the full optional/integration dependency set ``` +Pass extra pytest arguments through, e.g. `just test-optional -k test_attributes`, and select a Python version with `UV_PYTHON`, e.g. `UV_PYTHON=3.13 just test-optional`. + All tests are automatically run via GitHub Actions for every pull request and must pass before code can be accepted. Test coverage is also collected automatically via the Codecov service. > **Note:** Previous versions of Zarr-Python made extensive use of doctests. These tests were not maintained during the 3.0 refactor but may be brought back in the future. See issue #2614 for more details. +#### Minimum supported dependencies + +Zarr follows [SPEC 0](https://scientific-python.org/specs/spec-0000/) for the minimum versions of its dependencies. CI exercises those minimums in a dedicated `min_deps` job, which you can reproduce locally with: + +```bash +just min_deps +``` + +The supported floor for each runtime dependency is declared twice, and the two must be kept in sync: + +- the `>=` lower bound in `pyproject.toml` (`[project.dependencies]` and `[project.optional-dependencies]`) — what users actually get; and +- an exact pin in `ci/min-deps-constraints.txt` — what the `min_deps` job installs. + +When you raise a floor, update **both**. The `min_deps` job builds an isolated environment with `uv pip install --constraint ci/min-deps-constraints.txt`, which pins the runtime dependencies to their floors while letting transitive packages (e.g. flask, werkzeug) resolve to their latest compatible release. We pin the floors by hand rather than deriving them with uv's `--resolution lowest-direct` on purpose: hand-pinning tests the *exact* version we advertise (auto-derivation resolves to the lowest mutually consistent set, which can sit above the declared floor and hide a broken minimum), and it avoids dragging the committed `uv.lock` down. See the comments in `ci/min-deps-constraints.txt` for details. + ### Code standards - using prek All code must conform to the PEP8 standard. Regarding line length, lines up to 100 characters are allowed, although please try to keep under 90 wherever possible. @@ -190,15 +222,15 @@ If you would like to skip the failing checks and push the code for further discu Zarr strives to maintain 100% test coverage under the latest Python stable release. Both unit tests and docstring doctests are included when computing coverage. Running: ```bash -hatch env run --env test.py3.12-optional run-coverage +just test-optional ``` -will automatically run the test suite with coverage and produce an XML coverage report. This should be 100% before code can be accepted into the main code base. +will run the test suite with coverage and produce an XML coverage report. This should be 100% before code can be accepted into the main code base. -You can also generate an HTML coverage report by running: +You can also generate a browsable HTML coverage report by running: ```bash -hatch env run --env test.py3.12-optional run-coverage-html +just coverage-html ``` When submitting a pull request, coverage will also be collected across all supported Python versions via the Codecov service, and will be reported back within the pull request. Codecov coverage must also be 100% before code can be accepted. @@ -212,15 +244,15 @@ Zarr uses mkdocs for documentation, hosted on readthedocs.org. Documentation is The documentation can be built locally by running: ```bash -hatch --env docs run build +just docs-build ``` -The resulting built documentation will be available in the `docs/_build/html` folder. +The resulting built documentation will be available in the `site/` folder. -Hatch can also be used to serve continuously updating version of the documentation during development at [http://0.0.0.0:8000/](http://0.0.0.0:8000/). This can be done by running: +You can also serve a continuously updating version of the documentation during development at [http://0.0.0.0:8000/](http://0.0.0.0:8000/) by running: ```bash -hatch --env docs run serve +just docs-serve ``` #### Adding executable code blocks in the documentation diff --git a/pyproject.toml b/pyproject.toml index 02e66c67e8..b093bdb498 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,10 +30,14 @@ maintainers = [ { name = "Deepak Cherian" } ] requires-python = ">=3.12" +# The `>=` lower bounds below (and in [project.optional-dependencies]) are the +# minimum versions we support. The `min_deps` CI job tests them via the matching +# pins in `ci/min-deps-constraints.txt` — keep the two in sync when you bump a +# floor (per the SPEC 0 drop schedule: https://scientific-python.org/specs/spec-0000/). dependencies = [ 'packaging>=22.0', 'numpy>=2', - 'numcodecs>=0.14', + 'numcodecs>=0.14.1', 'google-crc32c>=1.5', 'typing_extensions>=4.14', 'donfig>=0.8', @@ -70,7 +74,7 @@ gpu = [ ] cast-value-rs = ["cast-value-rs"] cli = ["typer"] -optional = ["universal-pathlib"] +optional = ["universal-pathlib>=0.2.0"] [project.scripts] zarr = "zarr._cli.cli:app" @@ -83,15 +87,15 @@ Discussions = "https://github.com/zarr-developers/zarr-python/discussions" documentation = "https://zarr.readthedocs.io/" homepage = "https://github.com/zarr-developers/zarr-python" -# Dev *tooling* is pinned to exact versions for reproducible CI: the hatch envs -# (see `tool.hatch.envs.*`) and bare `uv run` resolve these groups fresh from -# PyPI and do NOT consult uv.lock, so an unrelated tooling release can break CI -# without any change on our side (e.g. the pytest 9.1.0 `duplicate -# parametrization` regression). Runtime/integration deps (fsspec, obstore, s3fs, -# botocore, numcodecs, universal-pathlib) are intentionally left floating so the -# `optional` test matrix keeps exercising their latest releases; their floor and -# bleeding edge are covered by the `min_deps` and `upstream` hatch envs. Bump the -# pins deliberately, e.g. via dependabot or `uv lock --upgrade`. +# Dev *tooling* is pinned to exact versions for reproducible CI, and the full +# transitive closure is locked in `uv.lock` (CI installs with `uv sync --locked`). +# Runtime/integration deps (fsspec, obstore, s3fs, botocore, numcodecs, +# universal-pathlib) instead carry floor (`>=`) constraints, so the `optional` +# test matrix keeps exercising their latest releases; their floor is covered by +# the `min_deps` CI job (hand-pinned in `ci/min-deps-constraints.txt`) and their +# bleeding edge by the `upstream` job (the `just upstream` recipe). Bump the +# floors/pins deliberately, e.g. via +# dependabot or `uv lock --upgrade`. [dependency-groups] test = [ "coverage==7.14.1", @@ -170,137 +174,10 @@ version.raw-options = { git_describe_command = "git describe --dirty --tags --lo [tool.hatch.build] hooks.vcs.version-file = "src/zarr/_version.py" -[tool.hatch.envs.dev] -dependency-groups = ["dev"] - -[tool.hatch.envs.test] -dependency-groups = ["test"] - -[tool.hatch.envs.test.env-vars] - -[[tool.hatch.envs.test.matrix]] -python = ["3.12", "3.13", "3.14"] -deps = ["minimal", "optional"] - -[tool.hatch.envs.test.overrides] -matrix.deps.features = [ - {value = "remote", if = ["optional"]}, - {value = "optional", if = ["optional"]}, - {value = "cli", if = ["optional"]}, - {value = "cast-value-rs", if = ["optional"]}, -] -matrix.deps.dependency-groups = [ - {value = "remote-tests", if = ["optional"]}, -] - -[tool.hatch.envs.test.scripts] -run-coverage = [ - "coverage run --source=src -m pytest --ignore tests/benchmarks --junitxml=junit.xml -o junit_family=legacy {args:}", - "coverage xml", -] -run-coverage-html = [ - "coverage run --source=src -m pytest --ignore tests/benchmarks {args:}", - "coverage html", -] -run = "pytest --ignore tests/benchmarks" -run-verbose = "run-coverage --verbose" -run-hypothesis = [ - "coverage run --source=src -m pytest -nauto --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful* {args:}", - "coverage xml", -] -run-benchmark = "pytest --benchmark-enable tests/benchmarks" -serve-coverage-html = "python -m http.server -d htmlcov 8000" -list-env = "pip list" - -[tool.hatch.envs.gputest] -template = "test" -extra-dependencies = [ - "universal_pathlib", - # Needed so tests/test_docs.py is collectable under `pytest -m gpu`; otherwise its - # module-level importorskip("pytest_examples") skips the whole module and the gpu - # docs example is never executed on GPU hardware. - "pytest-examples", -] -features = ["gpu"] - -[[tool.hatch.envs.gputest.matrix]] -python = ["3.12", "3.13"] - -[tool.hatch.envs.gputest.scripts] -run-coverage = [ - "coverage run --source=src -m pytest -m gpu --junitxml=junit.xml -o junit_family=legacy --ignore tests/benchmarks {args:}", - "coverage xml", -] -run = "pytest -m gpu --ignore tests/benchmarks" - -[tool.hatch.envs.upstream] -template = 'test' -python = "3.14" -extra-dependencies = [ - 'packaging @ git+https://github.com/pypa/packaging', - 'numpy', # from scientific-python-nightly-wheels - 'numcodecs @ git+https://github.com/zarr-developers/numcodecs', - 's3fs @ git+https://github.com/fsspec/s3fs', - 'universal_pathlib @ git+https://github.com/fsspec/universal_pathlib', - 'typing_extensions @ git+https://github.com/python/typing_extensions', - 'donfig @ git+https://github.com/pytroll/donfig', - 'obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore', -] - -[tool.hatch.envs.upstream.env-vars] -PIP_INDEX_URL = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/" -PIP_EXTRA_INDEX_URL = "https://pypi.org/simple/" -PIP_PRE = "1" - -[tool.hatch.envs.min_deps] -description = """Test environment for minimum supported dependencies - -See Spec 0000 for details and drop schedule: https://scientific-python.org/specs/spec-0000/ -""" -template = "test" -python = "3.12" -features = ["remote"] -dependency-groups = ["remote-tests"] -extra-dependencies = [ - 'packaging==22.*', - 'numpy==2.0.*', - 'numcodecs==0.14.*', # 0.14 needed for zarr3 codecs - 'fsspec==2023.10.0', - 's3fs==2023.10.0', - 'universal_pathlib==0.2.0', - 'typing_extensions==4.14.*', - 'donfig==0.8.*', - 'obstore==0.5.*', -] - -[tool.hatch.envs.default] -installer = "uv" - -[tool.hatch.envs.docs] -features = ['remote'] -dependency-groups = ['docs'] - -[tool.hatch.envs.docs.env-vars] -DISABLE_MKDOCS_2_WARNING = "true" -NO_MKDOCS_2_WARNING = "true" - -[tool.hatch.envs.docs.scripts] -serve = "mkdocs serve --watch src" -build = "mkdocs build" -check = "mkdocs build --strict" -readthedocs = "rm -rf $READTHEDOCS_OUTPUT/html && cp -r site $READTHEDOCS_OUTPUT/html" - -[tool.hatch.envs.doctest] -description = "Test environment for validating executable code blocks in documentation" -features = ['remote'] -dependency-groups = ['remote-tests'] -extra-dependencies = [ - "pytest-examples", -] - -[tool.hatch.envs.doctest.scripts] -test = "pytest tests/test_docs.py -v" -list-env = "pip list" +# NOTE: `tool.hatch.envs.*` used to define the test/docs/min_deps/upstream +# environments and their run scripts. Those are gone: tasks are now `just` +# recipes (see the Justfile) that wrap `uv sync --locked ...`; CI runs the same +# recipes via `uvx --from rust-just just `. See docs/contributing.md. [tool.ruff] line-length = 100 @@ -471,6 +348,7 @@ ignore = [ "PC170", # use PyGrep hooks - no *.rst files to check "PC180", # for JavaScript - not interested "PC902", # pre-commit.ci custom autofix message - not using autofix + "PY007", # task runner is a Justfile (just); not in PY007's nox/tox/hatch allowlist ] [tool.numpydoc_validation] diff --git a/tests/test_examples.py b/tests/test_examples.py index 9f8085e8c2..45d7d07160 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -58,7 +58,10 @@ def resave_script(source_path: Path, dest_path: Path) -> None: local Zarr project directory in the PEP-723 header. """ source_text = source_path.read_text() - dest_text = set_dep(source_text, f"zarr @ file:///{ZARR_PROJECT_PATH}") + # Use as_uri() to build a well-formed file URL. Naive f"file:///{abs_path}" + # yields four slashes (the path already starts with "/"), which strict + # packaging versions (our floor, 22.0) reject as an invalid URL. + dest_text = set_dep(source_text, f"zarr @ {ZARR_PROJECT_PATH.as_uri()}") dest_path.write_text(dest_text) diff --git a/uv.lock b/uv.lock index 799ea6e45a..6c230efa66 100644 --- a/uv.lock +++ b/uv.lock @@ -4078,13 +4078,13 @@ requires-dist = [ { name = "donfig", specifier = ">=0.8" }, { name = "fsspec", marker = "extra == 'remote'", specifier = ">=2023.10.0" }, { name = "google-crc32c", specifier = ">=1.5" }, - { name = "numcodecs", specifier = ">=0.14" }, + { name = "numcodecs", specifier = ">=0.14.1" }, { name = "numpy", specifier = ">=2" }, { name = "obstore", marker = "extra == 'remote'", specifier = ">=0.5.1" }, { name = "packaging", specifier = ">=22.0" }, { name = "typer", marker = "extra == 'cli'" }, { name = "typing-extensions", specifier = ">=4.14" }, - { name = "universal-pathlib", marker = "extra == 'optional'" }, + { name = "universal-pathlib", marker = "extra == 'optional'", specifier = ">=0.2.0" }, ] provides-extras = ["cast-value-rs", "cli", "gpu", "optional", "remote"]