From 0c679c84dbad6f6ab40a881676858a2643a60c04 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 11 Oct 2024 11:00:26 -0500 Subject: [PATCH 1/5] Special case `str` dtype in array creation (#2323) * Special case object dtype Closes https://github.com/zarr-developers/zarr-python/issues/2315 --------- Co-authored-by: Joe Hamman --- src/zarr/codecs/_v2.py | 2 +- src/zarr/core/array.py | 20 +++++++++++++++----- src/zarr/core/common.py | 14 ++++++++++++++ src/zarr/core/metadata/v2.py | 2 +- tests/v3/test_codecs/test_vlen.py | 2 +- tests/v3/test_v2.py | 21 +++++++++++++++++++-- 6 files changed, 51 insertions(+), 10 deletions(-) diff --git a/src/zarr/codecs/_v2.py b/src/zarr/codecs/_v2.py index 46fd1cb80..0f50264be 100644 --- a/src/zarr/codecs/_v2.py +++ b/src/zarr/codecs/_v2.py @@ -36,7 +36,7 @@ async def _decode_single( chunk_numpy_array = ensure_ndarray(chunk_bytes.as_array_like()) # ensure correct dtype - if str(chunk_numpy_array.dtype) != chunk_spec.dtype: + if str(chunk_numpy_array.dtype) != chunk_spec.dtype and not chunk_spec.dtype.hasobject: chunk_numpy_array = chunk_numpy_array.view(chunk_spec.dtype) return get_ndbuffer_class().from_numpy_array(chunk_numpy_array) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 5195db62a..ca405842e 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -35,6 +35,7 @@ ShapeLike, ZarrFormat, concurrent_map, + parse_dtype, parse_shapelike, product, ) @@ -365,16 +366,17 @@ async def create( ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: store_path = await make_store_path(store) + dtype_parsed = parse_dtype(dtype, zarr_format) shape = parse_shapelike(shape) if chunks is not None and chunk_shape is not None: raise ValueError("Only one of chunk_shape or chunks can be provided.") - dtype = np.dtype(dtype) if chunks: - _chunks = normalize_chunks(chunks, shape, dtype.itemsize) + _chunks = normalize_chunks(chunks, shape, dtype_parsed.itemsize) else: - _chunks = normalize_chunks(chunk_shape, shape, dtype.itemsize) + _chunks = normalize_chunks(chunk_shape, shape, dtype_parsed.itemsize) + result: AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] if zarr_format == 3: if dimension_separator is not None: @@ -396,7 +398,7 @@ async def create( result = await cls._create_v3( store_path, shape=shape, - dtype=dtype, + dtype=dtype_parsed, chunk_shape=_chunks, fill_value=fill_value, chunk_key_encoding=chunk_key_encoding, @@ -406,6 +408,14 @@ async def create( exists_ok=exists_ok, ) elif zarr_format == 2: + if dtype is str or dtype == "str": + # another special case: zarr v2 added the vlen-utf8 codec + vlen_codec: dict[str, JSON] = {"id": "vlen-utf8"} + if filters and not any(x["id"] == "vlen-utf8" for x in filters): + filters = list(filters) + [vlen_codec] + else: + filters = [vlen_codec] + if codecs is not None: raise ValueError( "codecs cannot be used for arrays with version 2. Use filters and compressor instead." @@ -419,7 +429,7 @@ async def create( result = await cls._create_v2( store_path, shape=shape, - dtype=dtype, + dtype=dtype_parsed, chunks=_chunks, dimension_separator=dimension_separator, fill_value=fill_value, diff --git a/src/zarr/core/common.py b/src/zarr/core/common.py index efce0f98f..0bc6245cb 100644 --- a/src/zarr/core/common.py +++ b/src/zarr/core/common.py @@ -14,6 +14,10 @@ overload, ) +import numpy as np + +from zarr.core.strings import _STRING_DTYPE + if TYPE_CHECKING: from collections.abc import Awaitable, Callable, Iterator @@ -151,3 +155,13 @@ def parse_order(data: Any) -> Literal["C", "F"]: if data in ("C", "F"): return cast(Literal["C", "F"], data) raise ValueError(f"Expected one of ('C', 'F'), got {data} instead.") + + +def parse_dtype(dtype: Any, zarr_format: ZarrFormat) -> np.dtype[Any]: + if dtype is str or dtype == "str": + if zarr_format == 2: + # special case as object + return np.dtype("object") + else: + return _STRING_DTYPE + return np.dtype(dtype) diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index d1dd86880..c5f34d277 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -321,7 +321,7 @@ def _default_fill_value(dtype: np.dtype[Any]) -> Any: """ if dtype.kind == "S": return b"" - elif dtype.kind == "U": + elif dtype.kind in "UO": return "" else: return dtype.type(0) diff --git a/tests/v3/test_codecs/test_vlen.py b/tests/v3/test_codecs/test_vlen.py index ca5ccb92f..aaea5dab8 100644 --- a/tests/v3/test_codecs/test_vlen.py +++ b/tests/v3/test_codecs/test_vlen.py @@ -11,7 +11,7 @@ from zarr.core.strings import _NUMPY_SUPPORTS_VLEN_STRING from zarr.storage.common import StorePath -numpy_str_dtypes: list[type | str | None] = [None, str, np.dtypes.StrDType] +numpy_str_dtypes: list[type | str | None] = [None, str, "str", np.dtypes.StrDType] expected_zarr_string_dtype: np.dtype[Any] if _NUMPY_SUPPORTS_VLEN_STRING: numpy_str_dtypes.append(np.dtypes.StringDType) diff --git a/tests/v3/test_v2.py b/tests/v3/test_v2.py index d981fbc89..729ed0533 100644 --- a/tests/v3/test_v2.py +++ b/tests/v3/test_v2.py @@ -2,6 +2,7 @@ from collections.abc import Iterator from typing import Any +import numcodecs.vlen import numpy as np import pytest from numcodecs import Delta @@ -44,7 +45,7 @@ def test_simple(store: StorePath) -> None: ("float64", 0.0), ("|S1", b""), ("|U1", ""), - ("object", 0), + ("object", ""), (str, ""), ], ) @@ -53,7 +54,12 @@ def test_implicit_fill_value(store: StorePath, dtype: str, fill_value: Any) -> N assert arr.metadata.fill_value is None assert arr.metadata.to_dict()["fill_value"] is None result = arr[:] - expected = np.full(arr.shape, fill_value, dtype=dtype) + if dtype is str: + # special case + numpy_dtype = np.dtype(object) + else: + numpy_dtype = np.dtype(dtype) + expected = np.full(arr.shape, fill_value, dtype=numpy_dtype) np.testing.assert_array_equal(result, expected) @@ -106,3 +112,14 @@ async def test_v2_encode_decode(dtype): data = zarr.open_array(store=store, path="foo")[:] expected = np.full((3,), b"X", dtype=dtype) np.testing.assert_equal(data, expected) + + +@pytest.mark.parametrize("dtype", [str, "str"]) +async def test_create_dtype_str(dtype: Any) -> None: + arr = zarr.create(shape=3, dtype=dtype, zarr_format=2) + assert arr.dtype.kind == "O" + assert arr.metadata.to_dict()["dtype"] == "|O" + assert arr.metadata.filters == (numcodecs.vlen.VLenUTF8(),) + arr[:] = ["a", "bb", "ccc"] + result = arr[:] + np.testing.assert_array_equal(result, np.array(["a", "bb", "ccc"], dtype="object")) From 42ab84761c53ebf39239b17795eb8421112f6a9c Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 11 Oct 2024 18:07:19 +0100 Subject: [PATCH 2/5] Add guide section to top level of docs (#2332) --- docs/{ => guide}/consolidated_metadata.rst | 0 docs/guide/index.rst | 7 +++++++ docs/index.rst | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) rename docs/{ => guide}/consolidated_metadata.rst (100%) create mode 100644 docs/guide/index.rst diff --git a/docs/consolidated_metadata.rst b/docs/guide/consolidated_metadata.rst similarity index 100% rename from docs/consolidated_metadata.rst rename to docs/guide/consolidated_metadata.rst diff --git a/docs/guide/index.rst b/docs/guide/index.rst new file mode 100644 index 000000000..106c35ce8 --- /dev/null +++ b/docs/guide/index.rst @@ -0,0 +1,7 @@ +Guide +===== + +.. toctree:: + :maxdepth: 1 + + consolidated_metadata diff --git a/docs/index.rst b/docs/index.rst index fd43d5e41..8e1b0740c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,7 +10,7 @@ Zarr-Python getting_started tutorial - consolidated_metadata + guide/index api/index spec release From 979b6f271bb6abaa55825bd12014807c727d3426 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 11 Oct 2024 16:08:53 -0500 Subject: [PATCH 3/5] Ensure paths created (#2337) --- src/zarr/storage/local.py | 5 +++++ tests/v3/test_store/test_local.py | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/zarr/storage/local.py b/src/zarr/storage/local.py index 1e1f3beec..da37cbfd5 100644 --- a/src/zarr/storage/local.py +++ b/src/zarr/storage/local.py @@ -94,6 +94,11 @@ def __init__(self, root: Path | str, *, mode: AccessModeLiteral = "r") -> None: assert isinstance(root, Path) self.root = root + async def _open(self) -> None: + if not self.mode.readonly: + self.root.mkdir(parents=True, exist_ok=True) + return await super()._open() + async def clear(self) -> None: self._check_writable() shutil.rmtree(self.root) diff --git a/tests/v3/test_store/test_local.py b/tests/v3/test_store/test_local.py index 8cdb4e874..5352e3520 100644 --- a/tests/v3/test_store/test_local.py +++ b/tests/v3/test_store/test_local.py @@ -1,11 +1,17 @@ from __future__ import annotations +from typing import TYPE_CHECKING + import pytest +import zarr from zarr.core.buffer import Buffer, cpu from zarr.storage.local import LocalStore from zarr.testing.store import StoreTests +if TYPE_CHECKING: + import pathlib + class TestLocalStore(StoreTests[LocalStore, cpu.Buffer]): store_cls = LocalStore @@ -40,3 +46,10 @@ async def test_empty_with_empty_subdir(self, store: LocalStore) -> None: assert await store.empty() (store.root / "foo/bar").mkdir(parents=True) assert await store.empty() + + def test_creates_new_directory(self, tmp_path: pathlib.Path): + target = tmp_path.joinpath("a", "b", "c") + assert not target.exists() + + store = self.store_cls(root=target, mode="w") + zarr.group(store=store) From ce5f53e3482727b7133def725af24576057aa164 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 11 Oct 2024 14:30:28 -0700 Subject: [PATCH 4/5] V3 main sync w/ merge (#2335) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump pypa/gh-action-pypi-publish from 1.8.10 to 1.8.11 (#1586) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.10 to 1.8.11. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.10...v1.8.11) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump conda-incubator/setup-miniconda from 2.3.0 to 3.0.1 (#1587) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 2.3.0 to 3.0.1. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v2.3.0...v3.0.1) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * * Cache result of FSStore._fsspec_installed (#1581) Prevent runtime-overhead in doing this check multiple times * Bump version of black in pre-commit (#1559) * Use list comprehension where applicable (#1555) Even if this is only a test, list comprehensions are faster than repeatedly call append(). Also use tuple instead of list when possible. Co-authored-by: Davis Bennett * Bump numcodecs from 0.11.0 to 0.12.1 (#1580) Bumps [numcodecs](https://github.com/zarr-developers/numcodecs) from 0.11.0 to 0.12.1. - [Release notes](https://github.com/zarr-developers/numcodecs/releases) - [Changelog](https://github.com/zarr-developers/numcodecs/blob/main/docs/release.rst) - [Commits](https://github.com/zarr-developers/numcodecs/compare/v0.11.0...v0.12.1) --- updated-dependencies: - dependency-name: numcodecs dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman * Use format specification mini-language to format string (#1558) Co-authored-by: Joe Hamman * Single startswith() call instead of multiple ones (#1556) It's faster and probably more readable. Co-authored-by: Davis Bennett Co-authored-by: Joe Hamman * Bump pymongo from 4.5.0 to 4.6.1 (#1585) Bumps [pymongo](https://github.com/mongodb/mongo-python-driver) from 4.5.0 to 4.6.1. - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/master/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/4.5.0...4.6.1) --- updated-dependencies: - dependency-name: pymongo dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman * Move codespell options around (#1196) Starting with codespell 2.2.2, options can be specified in `pyrpoject.toml` in addition to `setup.cfg`: https://github.com/codespell-project/codespell#using-a-config-file Specifying options in a config file instead of command line options in `.pre-commit-config.yaml` ensures codespell uses the same options when run as pre-commit hook or from the command line in the repository root directory. * Bump fsspec from 2023.10.0 to 2023.12.1 (#1600) * Bump fsspec from 2023.10.0 to 2023.12.1 Bumps [fsspec](https://github.com/fsspec/filesystem_spec) from 2023.10.0 to 2023.12.1. - [Commits](https://github.com/fsspec/filesystem_spec/compare/2023.10.0...2023.12.1) --- updated-dependencies: - dependency-name: fsspec dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update s3fs as well * Fix s3fs --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Josh Moore * Add type hints to zarr.create (#1536) * Add type hints to zarr.create * Use protocol for MetaArray * Use protocol for Synchronizer * Fix Path typing * Add release note * Fix dim separator typing * Ignore ... in coverage reporting * Fix chunk typing --------- Co-authored-by: Davis Bennett * Remove unused mypy ignore comments (#1602) Co-authored-by: Davis Bennett * Bump actions/setup-python from 4.7.1 to 5.0.0 (#1605) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.7.1 to 5.0.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.7.1...v5.0.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman * Bump github/codeql-action from 2 to 3 (#1609) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v2...v3) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore: update pre-commit hooks (#1448) * chore: update pre-commit hooks updates: - https://github.com/charliermarsh/ruff-pre-commit → https://github.com/astral-sh/ruff-pre-commit - [github.com/astral-sh/ruff-pre-commit: v0.0.224 → v0.1.8](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.224...v0.1.8) - [github.com/psf/black: 23.10.1 → 23.12.0](https://github.com/psf/black/compare/23.10.1...23.12.0) - [github.com/codespell-project/codespell: v2.2.5 → v2.2.6](https://github.com/codespell-project/codespell/compare/v2.2.5...v2.2.6) - [github.com/pre-commit/pre-commit-hooks: v4.4.0 → v4.5.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.4.0...v4.5.0) - [github.com/pre-commit/mirrors-mypy: v1.3.0 → v1.7.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.3.0...v1.7.1) * Attempt to fix ruff * Use isinstance --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Josh Moore * chore: update pre-commit hooks (#1618) updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.8 → v0.1.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.8...v0.1.9) - [github.com/psf/black: 23.12.0 → 23.12.1](https://github.com/psf/black/compare/23.12.0...23.12.1) - [github.com/pre-commit/mirrors-mypy: v1.7.1 → v1.8.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.7.1...v1.8.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Bump fsspec from 2023.12.1 to 2023.12.2 (#1606) * Bump fsspec from 2023.12.1 to 2023.12.2 Bumps [fsspec](https://github.com/fsspec/filesystem_spec) from 2023.12.1 to 2023.12.2. - [Commits](https://github.com/fsspec/filesystem_spec/compare/2023.12.1...2023.12.2) --- updated-dependencies: - dependency-name: fsspec dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update requirements_dev_optional.txt --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman * Bump pytest-doctestplus from 1.0.0 to 1.1.0 (#1619) Bumps [pytest-doctestplus](https://github.com/scientific-python/pytest-doctestplus) from 1.0.0 to 1.1.0. - [Release notes](https://github.com/scientific-python/pytest-doctestplus/releases) - [Changelog](https://github.com/scientific-python/pytest-doctestplus/blob/main/CHANGES.rst) - [Commits](https://github.com/scientific-python/pytest-doctestplus/compare/v1.0.0...v1.1.0) --- updated-dependencies: - dependency-name: pytest-doctestplus dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump pytest from 7.4.3 to 7.4.4 (#1622) * chore: update pre-commit hooks (#1626) updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.9 → v0.1.11](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.9...v0.1.11) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Create TEAM.md (#1628) * Drop python 3.8 and numpy 1.20 (#1557) * Drop 3.8 and add 3.12 * Try removing line_profiler * Also bump the minimal numpy to 1.21 * Drop 3.12 again * Revert "Try removing line_profiler" This reverts commit 837854bec99a9d25aece2ead9666f01690d228cc. * Update release.rst --------- Co-authored-by: Joe Hamman Co-authored-by: jakirkham * Add Norman Rzepka to core-dev team (#1630) * chore: update pre-commit hooks (#1633) updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.11 → v0.1.13](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.11...v0.1.13) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Bump actions/download-artifact from 3 to 4 (#1611) * Bump actions/download-artifact from 3 to 4 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 3 to 4. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Also bump upload-artifact see https://github.com/actions/download-artifact?tab=readme-ov-file#breaking-changes > Downloading artifacts that were created from action/upload-artifact@v3 and below are not supported. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman Co-authored-by: Josh Moore * Update tutorial.rst to include section about accessing Zip Files on S3 (#1615) * Update tutorial.rst to include section about accessing Zip Files on S3 Per discussion here, add information about about accessing zip files on s3: https://github.com/zarr-developers/zarr-python/discussions/1613 * Update release.rst * Implement d-v-b's suggestions --------- Co-authored-by: Davis Bennett Co-authored-by: Josh Moore * doc(v3): add v3 roadmap and design document (#1583) * doc(v3): add v3 roadmap and design document * Update v3-roadmap-and-design.md * updates after latest round of reviews * Update v3-roadmap-and-design.md Co-authored-by: Norman Rzepka * Update v3-roadmap-and-design.md Co-authored-by: Sanket Verma --------- Co-authored-by: Norman Rzepka Co-authored-by: Sanket Verma * chore: update pre-commit hooks (#1636) updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.13 → v0.1.14](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.13...v0.1.14) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Fix zarr sync (#1663) This patch removes fasteners and disables zarr.sync which uses process and thread Co-authored-by: Wei Ouyang * Update release.rst (#1621) * Update release.rst * Update release.rst * Change 2.16.2 → 2.17.0 * Update moto for test_s3 * Skip bsddb3 tests to prevent warning failure * Fix more user warning tests * Fix even more user warning tests * Skip coverage for importorskips * Move to have_X skip method for deps * Update release.rst (PR#1663) * Fix test_core.py 'compile' issues * Add black formatting * Drop Windows/3.9 build due to unrelated failures * fix typo --------- Co-authored-by: Davis Bennett Co-authored-by: Josh Moore * Bump numpy from 1.24.3 to 1.26.1 (#1543) Bumps [numpy](https://github.com/numpy/numpy) from 1.24.3 to 1.26.1. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.24.3...v1.26.1) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Davis Bennett Co-authored-by: Josh Moore Co-authored-by: Joe Hamman * chore: update pre-commit hooks (#1642) * chore: update pre-commit hooks updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.14 → v0.2.1](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.14...v0.2.1) - [github.com/psf/black: 23.12.1 → 24.2.0](https://github.com/psf/black/compare/23.12.1...24.2.0) * run black incl. comments for '...' --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Josh Moore * Bump ipywidgets from 8.1.0 to 8.1.1 (#1538) Bumps [ipywidgets](https://github.com/jupyter-widgets/ipywidgets) from 8.1.0 to 8.1.1. - [Release notes](https://github.com/jupyter-widgets/ipywidgets/releases) - [Commits](https://github.com/jupyter-widgets/ipywidgets/compare/8.1.0...8.1.1) --- updated-dependencies: - dependency-name: ipywidgets dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Davis Bennett Co-authored-by: Josh Moore * Proper argument for numpy.reshape (#1425) `numpy.reshape` not only accepts a tuple of ints, but also a simple int. Besides `(10)` is not a tuple and is identical to `10`, unlike `(10,)`. * Bump ipywidgets from 8.1.1 to 8.1.2 (#1666) Bumps [ipywidgets](https://github.com/jupyter-widgets/ipywidgets) from 8.1.1 to 8.1.2. - [Release notes](https://github.com/jupyter-widgets/ipywidgets/releases) - [Commits](https://github.com/jupyter-widgets/ipywidgets/compare/8.1.1...8.1.2) --- updated-dependencies: - dependency-name: ipywidgets dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * docs: ZIP-related tweaks (#1641) * docs: use 'ZIP archive' instead of 'zip file'; clarify utility of caching in s3 + ZIP example; style * docs: update release notes, correct spelling of greg lee's name in past release notes, and fix markup in past release notes * docs: use 'ZIP archive' instead of 'zip file'; clarify utility of caching in s3 + ZIP example; style * docs: update release notes, correct spelling of greg lee's name in past release notes, and fix markup in past release notes * Bump numpy from 1.26.1 to 1.26.4 (#1669) Bumps [numpy](https://github.com/numpy/numpy) from 1.26.1 to 1.26.4. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.26.1...v1.26.4) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Change occurrences of % and format() to f-strings (#1423) Co-authored-by: Joe Hamman Co-authored-by: Josh Moore * chore: update pre-commit hooks (#1672) updates: - [github.com/astral-sh/ruff-pre-commit: v0.2.1 → v0.2.2](https://github.com/astral-sh/ruff-pre-commit/compare/v0.2.1...v0.2.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Bump pymongo from 4.6.1 to 4.6.2 (#1674) Bumps [pymongo](https://github.com/mongodb/mongo-python-driver) from 4.6.1 to 4.6.2. - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/4.6.2/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/4.6.1...4.6.2) --- updated-dependencies: - dependency-name: pymongo dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump conda-incubator/setup-miniconda from 3.0.1 to 3.0.2 (#1677) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 3.0.1 to 3.0.2. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v3.0.1...v3.0.2) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Update config.yml with Zulip * Type dimension separator (#1620) Co-authored-by: Davis Bennett * Replace Gitter with new Zulip Chat link (#1685) * Replace Gitter with Zulip * Replace Gitter with Zulip in remaining places * Bump redis from 5.0.1 to 5.0.2 (#1688) Bumps [redis](https://github.com/redis/redis-py) from 5.0.1 to 5.0.2. - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v5.0.1...v5.0.2) --- updated-dependencies: - dependency-name: redis dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump pypa/gh-action-pypi-publish from 1.8.11 to 1.8.12 (#1691) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.11 to 1.8.12. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.11...v1.8.12) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump pytest-doctestplus from 1.1.0 to 1.2.0 (#1693) Bumps [pytest-doctestplus](https://github.com/scientific-python/pytest-doctestplus) from 1.1.0 to 1.2.0. - [Release notes](https://github.com/scientific-python/pytest-doctestplus/releases) - [Changelog](https://github.com/scientific-python/pytest-doctestplus/blob/main/CHANGES.rst) - [Commits](https://github.com/scientific-python/pytest-doctestplus/compare/v1.1.0...v1.2.0) --- updated-dependencies: - dependency-name: pytest-doctestplus dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Fix RTD build (#1694) * Update release.rst for v2.17.1 (#1673) * Update release.rst for v2.17.1 * Change the copyright year from 2023 → 2024. * Update release.rst for v2.17.1 * Bump pytest-timeout from 2.2.0 to 2.3.1 (#1697) Bumps [pytest-timeout](https://github.com/pytest-dev/pytest-timeout) from 2.2.0 to 2.3.1. - [Commits](https://github.com/pytest-dev/pytest-timeout/compare/2.2.0...2.3.1) --- updated-dependencies: - dependency-name: pytest-timeout dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump conda-incubator/setup-miniconda from 3.0.2 to 3.0.3 (#1690) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 3.0.2 to 3.0.3. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v3.0.2...v3.0.3) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sanket Verma * docs(tutorial.rst): fix link to GCSMap (#1689) * Update installation.rst stating version support policy (#1665) * Update installation.rst stating version support policy * Update docs/installation.rst Co-authored-by: Joe Hamman * Update docs/installation.rst --------- Co-authored-by: Joe Hamman * Bump pypa/gh-action-pypi-publish from 1.8.12 to 1.8.14 (#1700) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.12 to 1.8.14. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.12...v1.8.14) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump pytest-doctestplus from 1.2.0 to 1.2.1 (#1699) Bumps [pytest-doctestplus](https://github.com/scientific-python/pytest-doctestplus) from 1.2.0 to 1.2.1. - [Release notes](https://github.com/scientific-python/pytest-doctestplus/releases) - [Changelog](https://github.com/scientific-python/pytest-doctestplus/blob/main/CHANGES.rst) - [Commits](https://github.com/scientific-python/pytest-doctestplus/compare/v1.2.0...v1.2.1) --- updated-dependencies: - dependency-name: pytest-doctestplus dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sanket Verma * Bump redis from 5.0.2 to 5.0.3 (#1698) Bumps [redis](https://github.com/redis/redis-py) from 5.0.2 to 5.0.3. - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v5.0.2...v5.0.3) --- updated-dependencies: - dependency-name: redis dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sanket Verma * Add Python 3.12 to CI (#1719) * Update python-package.yml * bump numpy versions * bump min python version * Update release.rst * Bump pytest-cov from 4.1.0 to 5.0.0 (#1722) Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 4.1.0 to 5.0.0. - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v4.1.0...v5.0.0) --- updated-dependencies: - dependency-name: pytest-cov dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore: update pre-commit hooks (#1708) updates: - [github.com/astral-sh/ruff-pre-commit: v0.2.2 → v0.3.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.2.2...v0.3.3) - [github.com/psf/black: 24.2.0 → 24.3.0](https://github.com/psf/black/compare/24.2.0...24.3.0) - [github.com/pre-commit/mirrors-mypy: v1.8.0 → v1.9.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.8.0...v1.9.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Sanket Verma * chore: update pre-commit hooks (#1723) updates: - [github.com/astral-sh/ruff-pre-commit: v0.3.3 → v0.3.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.3...v0.3.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Fix release notes (following #1719) (#1725) * Override ipython repr methods. (#1724) Closes #1716 This avoids expensive lookups against object stores. * Bump pymongo from 4.6.2 to 4.6.3 (#1729) * Remove v1 and v2 specification (#1582) * Remove v1 and v2 specification * fix warning --------- Co-authored-by: Davis Bennett * chore: update pre-commit hooks (#1738) updates: - [github.com/astral-sh/ruff-pre-commit: v0.3.4 → v0.3.5](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.4...v0.3.5) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Optimize Array.info and Group.info (#1733) * Optimize Array.info. Avoid repeated computes of the same value (getsize) * Don't have InfoReporter query items twice. Apparently IPython will run both __repr__ and _repr_html_ so we were calling `getsize` twice. * Group too * Apply suggestions from code review Co-authored-by: Joe Hamman --------- Co-authored-by: Joe Hamman * Bump actions/setup-python from 5.0.0 to 5.1.0 (#1736) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.0.0 to 5.1.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.0.0...v5.1.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Couple fixes (#1737) * Use `is` when comparing `type` of two objects * Unnecessary `None` provided as default * Fix tests with Pytest 8 (#1714) * Bump pytest version * Use editable install when testing --------- Co-authored-by: Sanket Verma * Avoid redundant __contains__ (#1739) Let's try grabbing the array.json and group.json files, and check for `*NotFoundError`, instead of using contains first. Co-authored-by: Davis Bennett * Array & Group: Use already loaded attributes to populate cache. (#1734) * Array: Use already loaded attributes to populate cache. * Group: Use already loaded attributes to populate cache. * Fix * Add release note * Optimize attribute setting (#1741) * Optimize attribute setting * Add release note --------- Co-authored-by: Davis Bennett * Make sure fs exceptions are raised if not MissingFs exceptions (clone) (#1604) * Make sure fs exceptions are raised if not Missing * lint * add missing argument in tests, lint * clear memory filesystem during test * improve commenting * add memory_store fixture, getitems performance * Update release.rst * improve FSStore.test_exception coverage --------- Co-authored-by: Martin Durant Co-authored-by: Joe Hamman Co-authored-by: Josh Moore Co-authored-by: Sanket Verma * chore(release): update changelog for 2.17.2 (#1775) * chore(docs): reset release notes as unreleased (#1776) * Bump codecov/codecov-action from 3 to 4 (#1647) * Bump codecov/codecov-action from 3 to 4 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3 to 4. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3...v4) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Set codecov env --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Josh Moore * Update release.rst for v2.17.2 (#1778) * Update release.rst for v2.17.2 * Minor edits --------- Co-authored-by: Joe Hamman * Deprecate the experimental v3 implementation (#1802) * deprecate(exp-v3): Add a future warning about the pending removal of the experimental v3 implementation * ignore warning * add test * chore: update pre-commit hooks (#1779) updates: - [github.com/astral-sh/ruff-pre-commit: v0.3.5 → v0.3.7](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.5...v0.3.7) - [github.com/psf/black: 24.3.0 → 24.4.0](https://github.com/psf/black/compare/24.3.0...24.4.0) - [github.com/pre-commit/pre-commit-hooks: v4.5.0 → v4.6.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.5.0...v4.6.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Fix `is_total_slice` for size-1 dimensions (#1800) Closes #1730 Co-authored-by: Ryan Abernathey Co-authored-by: Joe Hamman * add note to the top of the release page noting the plan for 2.18.* and 3.0 (#1816) * Bump conda-incubator/setup-miniconda from 3.0.3 to 3.0.4 (#1824) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 3.0.3 to 3.0.4. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v3.0.3...v3.0.4) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * dep(docs): deprecate experimental v3 support in docs (#1807) * dep(docs): deprecate experimental v3 support in docs * Apply suggestions from code review Co-authored-by: Josh Moore Co-authored-by: Sanket Verma --------- Co-authored-by: Josh Moore Co-authored-by: Sanket Verma * Bump h5py from 3.10.0 to 3.11.0 (#1786) Bumps [h5py](https://github.com/h5py/h5py) from 3.10.0 to 3.11.0. - [Release notes](https://github.com/h5py/h5py/releases) - [Changelog](https://github.com/h5py/h5py/blob/master/docs/release_guide.rst) - [Commits](https://github.com/h5py/h5py/compare/3.10.0...3.11.0) --- updated-dependencies: - dependency-name: h5py dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman * Bump redis from 5.0.3 to 5.0.4 (#1810) Bumps [redis](https://github.com/redis/redis-py) from 5.0.3 to 5.0.4. - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v5.0.3...v5.0.4) --- updated-dependencies: - dependency-name: redis dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * deprecate(stores): add deprecation warnings to stores that we plan to remove in v3 (#1801) * deprecate(stores): add deprecation warnings to DBMStore, LMDBStore, SQLiteStore, MongoDBStore, RedisStore, and ABSStore * filter warnings in pytest config * more deprecation warnings in docstrings * add release note * use np.inf instead of PINF/NINF (#1842) * use np.inf instead of PINF/NINF * update release notes * chore: update pre-commit hooks (#1825) updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.1 → v0.4.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.1...v0.4.3) - [github.com/psf/black: 24.4.0 → 24.4.2](https://github.com/psf/black/compare/24.4.0...24.4.2) - [github.com/pre-commit/mirrors-mypy: v1.9.0 → v1.10.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.9.0...v1.10.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Enable ruff/bugbear rules (B) and fix issues (#1702) * Enable ruff/bugbear rules (B) As suggested by Repo-Review. * Fix ruff/bugbear issue (B007) B007 Loop control variable `key` not used within loop body https://docs.astral.sh/ruff/rules/unused-loop-control-variable/ * Fix ruff/bugbear issue (B015) B015 Pointless comparison. Did you mean to assign a value? Otherwise, prepend `assert` or remove it. https://docs.astral.sh/ruff/rules/useless-comparison/ * Fix ruff/bugbear issues (B028) B028 No explicit `stacklevel` keyword argument found https://docs.astral.sh/ruff/rules/no-explicit-stacklevel/ * Fix ruff/bugbear issues (B904) B904 Within an `except` clause, raise exceptions with `raise ... from err` or `raise ... from None` to distinguish them from errors in exception handling https://docs.astral.sh/ruff/rules/raise-without-from-inside-except/ * Document changes in docs/release.rst * Disable ruff/bugbear rule (B017) B017 `pytest.raises(Exception)` should be considered evil https://docs.astral.sh/ruff/rules/assert-raises-exception/ --------- Co-authored-by: Joe Hamman * Release notes for 2.18.0 (#1843) * doc: cleanup release notes for 2.18.0 * Update release.rst (#1850) * Group dependabot updates (#1854) * chore: update pre-commit hooks (#1876) updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.3 → v0.4.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.3...v0.4.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Fix a regression with scalar indexing due to #1800 (#1875) * release notes for 2.18.1 (#1885) * reset release notes (#1886) * Add zstd to old V3 supported codecs (#1914) * add zstd to old V3 supported codecs * get to full test coverage and add release note * fix pre-commit * doc: update release notes for 2.18.2 (#1915) * Drop support for Python 3.9 * Revert "Drop support for Python 3.9" This reverts commit a054afbbdaf7c8560d0ce5df77af969bc863e335. * Update TEAM.md (#2071) * [v2] Fix doctests with numpy 2.0 (#2073) * Fix version number in built docs (#2044) * Fix orthogonal indexing with scalar. (#1947) Co-authored-by: David Stansby * Bump the requirements group across 1 directory with 7 updates (#2092) Bumps the requirements group with 7 updates in the / directory: | Package | From | To | | --- | --- | --- | | [numpy](https://github.com/numpy/numpy) | `1.26.4` | `2.0.1` | | [ipywidgets](https://github.com/jupyter-widgets/ipywidgets) | `8.1.2` | `8.1.3` | | [setuptools-scm](https://github.com/pypa/setuptools_scm) | `8.0.4` | `8.1.0` | | [pytest](https://github.com/pytest-dev/pytest) | `8.1.1` | `8.3.2` | | [lmdb](https://github.com/jnwatson/py-lmdb) | `1.4.1` | `1.5.1` | | [redis](https://github.com/redis/redis-py) | `5.0.4` | `5.0.8` | | [pymongo](https://github.com/mongodb/mongo-python-driver) | `4.6.3` | `4.8.0` | Updates `numpy` from 1.26.4 to 2.0.1 - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.26.4...v2.0.1) Updates `ipywidgets` from 8.1.2 to 8.1.3 - [Release notes](https://github.com/jupyter-widgets/ipywidgets/releases) - [Commits](https://github.com/jupyter-widgets/ipywidgets/compare/8.1.2...8.1.3) Updates `setuptools-scm` from 8.0.4 to 8.1.0 - [Release notes](https://github.com/pypa/setuptools_scm/releases) - [Changelog](https://github.com/pypa/setuptools_scm/blob/main/CHANGELOG.md) - [Commits](https://github.com/pypa/setuptools_scm/compare/v8.0.4...v8.1.0) Updates `pytest` from 8.1.1 to 8.3.2 - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.1.1...8.3.2) Updates `lmdb` from 1.4.1 to 1.5.1 - [Changelog](https://github.com/jnwatson/py-lmdb/blob/master/ChangeLog) - [Commits](https://github.com/jnwatson/py-lmdb/compare/py-lmdb_1.4.1...py-lmdb_1.5.1) Updates `redis` from 5.0.4 to 5.0.8 - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v5.0.4...v5.0.8) Updates `pymongo` from 4.6.3 to 4.8.0 - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/master/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/4.6.3...4.8.0) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:development update-type: version-update:semver-major dependency-group: requirements - dependency-name: ipywidgets dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements - dependency-name: setuptools-scm dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements - dependency-name: lmdb dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements - dependency-name: redis dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements - dependency-name: pymongo dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump the actions group with 2 updates (#2087) Bumps the actions group with 2 updates: [actions/setup-python](https://github.com/actions/setup-python) and [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `actions/setup-python` from 5.1.0 to 5.1.1 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.1.0...v5.1.1) Updates `pypa/gh-action-pypi-publish` from 1.8.14 to 1.9.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.14...v1.9.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * [v2] Drop support for Python 3.9 (#2074) * Drop support for Python 3.9 * Ignore B905 ruff rule * Fix release notes * Fix Array.__array__ for numpy 2.1 (#2106) * Fix Array.__array__ for numpy 2.1 * Add changelog * Depend on np.array for array coercions * Bump test version of numcodecs (#2114) * Bump test version of numcodecs * Fix test for numcodecs 0.13 * fix: numpy 1.24 compat for Array.__array__ (#2123) * Deprecate N5Store (#2103) * deprecate(n5): add deprecation warning to N5Store * also deprecate N5FSStore * docs * fix doc * Run tests on numpy 1.23 (#2124) * Bump the requirements group across 1 directory with 3 updates (#2129) Bumps the requirements group with 3 updates in the / directory: [numpy](https://github.com/numpy/numpy), [ipywidgets](https://github.com/jupyter-widgets/ipywidgets) and [azure-storage-blob](https://github.com/Azure/azure-sdk-for-python). Updates `numpy` from 2.0.1 to 2.1.0 - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v2.0.1...v2.1.0) Updates `ipywidgets` from 8.1.3 to 8.1.5 - [Release notes](https://github.com/jupyter-widgets/ipywidgets/releases) - [Commits](https://github.com/jupyter-widgets/ipywidgets/compare/8.1.3...8.1.5) Updates `azure-storage-blob` from 12.16.0 to 12.21.0 - [Release notes](https://github.com/Azure/azure-sdk-for-python/releases) - [Changelog](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/esrp_release.md) - [Commits](https://github.com/Azure/azure-sdk-for-python/compare/azure-storage-blob_12.16.0...azure-storage-blob_12.21.0) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements - dependency-name: ipywidgets dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements - dependency-name: azure-storage-blob dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore: bump minimum numpy version to 1.24 (#2127) * chore: bump minimum version to 1.24 * Update .github/workflows/python-package.yml * Remove un-needed package installs in CI (#2095) * Bump the actions group with 2 updates (#2146) Bumps the actions group with 2 updates: [actions/setup-python](https://github.com/actions/setup-python) and [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `actions/setup-python` from 5.1.1 to 5.2.0 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.1.1...v5.2.0) Updates `pypa/gh-action-pypi-publish` from 1.9.0 to 1.10.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.9.0...v1.10.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump numpy from 2.1.0 to 2.1.1 in the requirements group (#2151) Bumps the requirements group with 1 update: [numpy](https://github.com/numpy/numpy). Updates `numpy` from 2.1.0 to 2.1.1 - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v2.1.0...v2.1.1) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(docs): update release notes ahead of 2.18.4 (#2152) * Bump pypa/gh-action-pypi-publish from 1.10.0 to 1.10.1 in the actions group (#2161) Bumps the actions group with 1 update: [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `pypa/gh-action-pypi-publish` from 1.10.0 to 1.10.1 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.10.0...v1.10.1) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * [docs] remove primary sidebar from tutorial (#2142) * remove primary sidebar from tutorial * lint * Bump pytest from 8.3.2 to 8.3.3 in the requirements group (#2172) Bumps the requirements group with 1 update: [pytest](https://github.com/pytest-dev/pytest). Updates `pytest` from 8.3.2 to 8.3.3 - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.3.2...8.3.3) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump pypa/gh-action-pypi-publish in the actions group (#2220) Bumps the actions group with 1 update: [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `pypa/gh-action-pypi-publish` from 1.10.1 to 1.10.2 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.10.1...v1.10.2) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Update TEAM.md (#2227) * Bump the requirements group across 1 directory with 3 updates (#2284) Bumps the requirements group with 3 updates in the / directory: [redis](https://github.com/redis/redis-py), [pymongo](https://github.com/mongodb/mongo-python-driver) and [h5py](https://github.com/h5py/h5py). Updates `redis` from 5.0.8 to 5.1.0 - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v5.0.8...v5.1.0) Updates `pymongo` from 4.8.0 to 4.10.0 - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/master/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/4.8.0...4.10.0) Updates `h5py` from 3.11.0 to 3.12.1 - [Release notes](https://github.com/h5py/h5py/releases) - [Changelog](https://github.com/h5py/h5py/blob/master/docs/release_guide.rst) - [Commits](https://github.com/h5py/h5py/compare/3.11.0...3.12.1) --- updated-dependencies: - dependency-name: redis dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements - dependency-name: pymongo dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements - dependency-name: h5py dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump pymongo from 4.10.0 to 4.10.1 in the requirements group (#2288) Bumps the requirements group with 1 update: [pymongo](https://github.com/mongodb/mongo-python-driver). Updates `pymongo` from 4.10.0 to 4.10.1 - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/master/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/4.10.0...4.10.1) --- updated-dependencies: - dependency-name: pymongo dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Fix low contrast box titles (#2287) (#2292) * Bump pypa/gh-action-pypi-publish in the actions group (#2304) Bumps the actions group with 1 update: [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `pypa/gh-action-pypi-publish` from 1.10.2 to 1.10.3 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.10.2...v1.10.3) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump the requirements group with 2 updates (#2303) Bumps the requirements group with 2 updates: [numpy](https://github.com/numpy/numpy) and [redis](https://github.com/redis/redis-py). Updates `numpy` from 2.1.1 to 2.1.2 - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v2.1.1...v2.1.2) Updates `redis` from 5.1.0 to 5.1.1 - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v5.1.0...v5.1.1) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements - dependency-name: redis dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump numcodecs from 0.13.0 to 0.13.1 in the requirements group (#2326) Bumps the requirements group with 1 update: [numcodecs](https://github.com/zarr-developers/numcodecs). Updates `numcodecs` from 0.13.0 to 0.13.1 - [Release notes](https://github.com/zarr-developers/numcodecs/releases) - [Changelog](https://github.com/zarr-developers/numcodecs/blob/main/docs/release.rst) - [Commits](https://github.com/zarr-developers/numcodecs/compare/v0.13.0...v0.13.1) --- updated-dependencies: - dependency-name: numcodecs dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * remove zarr.types from v2 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Janick Martinez Esturo Co-authored-by: David Stansby Co-authored-by: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Co-authored-by: Davis Bennett Co-authored-by: Josh Moore Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: jakirkham Co-authored-by: Jeff Peck Co-authored-by: Norman Rzepka Co-authored-by: Sanket Verma Co-authored-by: Hood Chatham Co-authored-by: Wei Ouyang Co-authored-by: Daniel Jahn (dahn) Co-authored-by: Deepak Cherian Co-authored-by: Ian Carroll Co-authored-by: Martin Durant Co-authored-by: Ryan Abernathey Co-authored-by: Jonny Saunders --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/releases.yml | 2 +- LICENSE.txt | 2 +- README.md | 6 +- TEAM.md | 29 ++ docs/conf.py | 16 +- docs/index.rst | 4 +- docs/installation.rst | 5 + docs/release.rst | 33 ++ docs/spec/v1.rst | 267 +----------- docs/spec/v2.rst | 562 +------------------------- docs/spec/v3.rst | 2 +- docs/tutorial.rst | 40 +- pyproject.toml | 1 + v3-roadmap-and-design.md | 429 ++++++++++++++++++++ 15 files changed, 549 insertions(+), 851 deletions(-) create mode 100644 TEAM.md create mode 100644 v3-roadmap-and-design.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 2e11f1461..705cd31cb 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -27,7 +27,7 @@ body: attributes: label: Python Version description: Version of Python interpreter - placeholder: 3.8.5, 3.9, 3.10, etc. + placeholder: 3.10, 3.11, 3.12 etc. validations: required: true - type: input diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index f694039ce..bab53958d 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -16,7 +16,7 @@ jobs: submodules: true fetch-depth: 0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5.2.0 name: Install Python with: python-version: '3.11' diff --git a/LICENSE.txt b/LICENSE.txt index 850a0d877..a4de1c39d 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015-2023 Zarr Developers +Copyright (c) 2015-2024 Zarr Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index b035ffa59..e379c9719 100644 --- a/README.md +++ b/README.md @@ -70,10 +70,10 @@ - Gitter + Zulip - - + + diff --git a/TEAM.md b/TEAM.md new file mode 100644 index 000000000..e6975d7c0 --- /dev/null +++ b/TEAM.md @@ -0,0 +1,29 @@ +## Active core-developers +- @joshmoore (Josh Moore) +- @jni (Juan Nunez-Iglesias) +- @rabernat (Ryan Abernathey) +- @jhamman (Joe Hamman) +- @d-v-b (Davis Bennett) +- @jakirkham (jakirkham) +- @martindurant (Martin Durant) +- @normanrz (Norman Rzepka) +- @dstansby (David Stansby) +- @dcherian (Deepak Cherian) +- @TomAugspurger (Tom Augspurger) + +## Emeritus core-developers +- @alimanfoo (Alistair Miles) +- @shoyer (Stephan Hoyer) +- @ryan-williams (Ryan Williams) +- @jrbourbeau (James Bourbeau) +- @mzjp2 (Zain Patel) +- @grlee77 (Gregory Lee) + +## Former core-developers +- @jeromekelleher (Jerome Kelleher) +- @tjcrone (Tim Crone) +- @funkey (Jan Funke) +- @shikharsg +- @Carreau (Matthias Bussonnier) +- @dazzag24 +- @WardF (Ward Fisher) diff --git a/docs/conf.py b/docs/conf.py index ec8509837..0a328ac25 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,11 +19,8 @@ import sphinx.application -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. +from importlib.metadata import version as get_version + import zarr import sphinx @@ -79,12 +76,11 @@ # General information about the project. project = "zarr" -copyright = "2023, Zarr Developers" +copyright = "2024, Zarr Developers" author = "Zarr Developers" -version = zarr.__version__ -# The full version, including alpha/beta/rc tags. -release = zarr.__version__ +version = get_version("zarr") +release = get_version("zarr") # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -205,7 +201,7 @@ def setup(app: sphinx.application.Sphinx) -> None: # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -# html_sidebars = {} +html_sidebars = {"tutorial": []} # Additional templates that should be rendered to pages, maps page names to # template names. diff --git a/docs/index.rst b/docs/index.rst index 8e1b0740c..6b90b5a77 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,13 +19,13 @@ Zarr-Python **Version**: |version| -**Download documentation**: `PDF/Zipped HTML/EPUB `_ +**Download documentation**: `PDF/Zipped HTML `_ **Useful links**: `Installation `_ | `Source Repository `_ | `Issue Tracker `_ | -`Gitter `_ +`Zulip Chat `_ Zarr is a file storage format for chunked, compressed, N-dimensional arrays based on an open-source specification. diff --git a/docs/installation.rst b/docs/installation.rst index 3d4ac4107..86da6d103 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -6,6 +6,11 @@ Zarr depends on NumPy. It is generally best to `install NumPy appropriate for your operating system and Python distribution. Other dependencies should be installed automatically if using one of the installation methods below. +Note: Zarr has endorsed `Scientific-Python SPEC 0 `_ and now follows the version support window as outlined below: + +- Python: 36 months after initial release +- Core package dependencies (e.g. NumPy): 24 months after initial release + Install Zarr from PyPI:: $ pip install zarr diff --git a/docs/release.rst b/docs/release.rst index 5c3a43a14..0b6775c4a 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -303,6 +303,39 @@ Documentation * doc: update release notes for 3.0.0.alpha. By :user:`Joe Hamman ` :issue:`1959`. +.. _release_2.18.3: + +2.18.3 +------ + +Enhancements +~~~~~~~~~~~~ +* Added support for creating a copy of data when converting a `zarr.Array` + to a numpy array. + By :user:`David Stansby ` (:issue:`2106`) and + :user:`Joe Hamman ` (:issue:`2123`). + +Maintenance +~~~~~~~~~~~ +* Removed support for Python 3.9. + By :user:`David Stansby ` (:issue:`2074`). + +* Fix a regression when using orthogonal indexing with a scalar. + By :user:`Deepak Cherian ` :issue:`1931` + +* Added compatibility with NumPy 2.1. + By :user:`David Stansby ` + +* Bump minimum NumPy version to 1.24. + :user:`Joe Hamman ` (:issue:`2127`). + +Deprecations +~~~~~~~~~~~~ + +* Deprecate :class:`zarr.n5.N5Store` and :class:`zarr.n5.N5FSStore`. These + stores are slated to be removed in Zarr Python 3.0. + By :user:`Joe Hamman ` :issue:`2085`. + .. _release_2.18.2: 2.18.2 diff --git a/docs/spec/v1.rst b/docs/spec/v1.rst index 13f68ef36..27a0490e0 100644 --- a/docs/spec/v1.rst +++ b/docs/spec/v1.rst @@ -3,268 +3,5 @@ Zarr Storage Specification Version 1 ==================================== -This document provides a technical specification of the protocol and -format used for storing a Zarr array. The key words "MUST", "MUST -NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", -"RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be -interpreted as described in `RFC 2119 -`_. - -Status ------- - -This specification is deprecated. See :ref:`spec` for the latest version. - -Storage -------- - -A Zarr array can be stored in any storage system that provides a -key/value interface, where a key is an ASCII string and a value is an -arbitrary sequence of bytes, and the supported operations are read -(get the sequence of bytes associated with a given key), write (set -the sequence of bytes associated with a given key) and delete (remove -a key/value pair). - -For example, a directory in a file system can provide this interface, -where keys are file names, values are file contents, and files can be -read, written or deleted via the operating system. Equally, an S3 -bucket can provide this interface, where keys are resource names, -values are resource contents, and resources can be read, written or -deleted via HTTP. - -Below an "array store" refers to any system implementing this -interface. - -Metadata --------- - -Each array requires essential configuration metadata to be stored, -enabling correct interpretation of the stored data. This metadata is -encoded using JSON and stored as the value of the 'meta' key within an -array store. - -The metadata resource is a JSON object. The following keys MUST be -present within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. -shape - A list of integers defining the length of each dimension of the array. -chunks - A list of integers defining the length of each dimension of a chunk of the - array. Note that all chunks within a Zarr array have the same shape. -dtype - A string or list defining a valid data type for the array. See also - the subsection below on data type encoding. -compression - A string identifying the primary compression library used to compress - each chunk of the array. -compression_opts - An integer, string or dictionary providing options to the primary - compression library. -fill_value - A scalar value providing the default value to use for uninitialized - portions of the array. -order - Either 'C' or 'F', defining the layout of bytes within each chunk of the - array. 'C' means row-major order, i.e., the last dimension varies fastest; - 'F' means column-major order, i.e., the first dimension varies fastest. - -Other keys MAY be present within the metadata object however they MUST -NOT alter the interpretation of the required fields defined above. - -For example, the JSON object below defines a 2-dimensional array of -64-bit little-endian floating point numbers with 10000 rows and 10000 -columns, divided into chunks of 1000 rows and 1000 columns (so there -will be 100 chunks in total arranged in a 10 by 10 grid). Within each -chunk the data are laid out in C contiguous order, and each chunk is -compressed using the Blosc compression library:: - - { - "chunks": [ - 1000, - 1000 - ], - "compression": "blosc", - "compression_opts": { - "clevel": 5, - "cname": "lz4", - "shuffle": 1 - }, - "dtype": "`_. The -format consists of 3 parts: a character describing the byteorder of -the data (``<``: little-endian, ``>``: big-endian, ``|``: -not-relevant), a character code giving the basic type of the array, -and an integer providing the number of bytes the type uses. The byte -order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and -``"|S12"`` are valid data types. - -Structure data types (i.e., with multiple named fields) are encoded as -a list of two-element lists, following `NumPy array protocol type -descriptions (descr) -`_. -For example, the JSON list ``[["r", "|u1"], ["g", "|u1"], ["b", -"|u1"]]`` defines a data type composed of three single-byte unsigned -integers labelled 'r', 'g' and 'b'. - -Chunks ------- - -Each chunk of the array is compressed by passing the raw bytes for the -chunk through the primary compression library to obtain a new sequence -of bytes comprising the compressed chunk data. No header is added to -the compressed bytes or any other modification made. The internal -structure of the compressed bytes will depend on which primary -compressor was used. For example, the `Blosc compressor -`_ -produces a sequence of bytes that begins with a 16-byte header -followed by compressed data. - -The compressed sequence of bytes for each chunk is stored under a key -formed from the index of the chunk within the grid of chunks -representing the array. To form a string key for a chunk, the indices -are converted to strings and concatenated with the period character -('.') separating each index. For example, given an array with shape -(10000, 10000) and chunk shape (1000, 1000) there will be 100 chunks -laid out in a 10 by 10 grid. The chunk with indices (0, 0) provides -data for rows 0-999 and columns 0-999 and is stored under the key -'0.0'; the chunk with indices (2, 4) provides data for rows 2000-2999 -and columns 4000-4999 and is stored under the key '2.4'; etc. - -There is no need for all chunks to be present within an array -store. If a chunk is not present then it is considered to be in an -uninitialized state. An uninitialized chunk MUST be treated as if it -was uniformly filled with the value of the 'fill_value' field in the -array metadata. If the 'fill_value' field is ``null`` then the -contents of the chunk are undefined. - -Note that all chunks in an array have the same shape. If the length of -any array dimension is not exactly divisible by the length of the -corresponding chunk dimension then some chunks will overhang the edge -of the array. The contents of any chunk region falling outside the -array are undefined. - -Attributes ----------- - -Each array can also be associated with custom attributes, which are -simple key/value items with application-specific meaning. Custom -attributes are encoded as a JSON object and stored under the 'attrs' -key within an array store. Even if the attributes are empty, the -'attrs' key MUST be present within an array store. - -For example, the JSON object below encodes three attributes named -'foo', 'bar' and 'baz':: - - { - "foo": 42, - "bar": "apples", - "baz": [1, 2, 3, 4] - } - -Example -------- - -Below is an example of storing a Zarr array, using a directory on the -local file system as storage. - -Initialize the store:: - - >>> import zarr - >>> store = zarr.DirectoryStore('example.zarr') - >>> zarr.init_store(store, shape=(20, 20), chunks=(10, 10), - ... dtype='i4', fill_value=42, compression='zlib', - ... compression_opts=1, overwrite=True) - -No chunks are initialized yet, so only the 'meta' and 'attrs' keys -have been set:: - - >>> import os - >>> sorted(os.listdir('example.zarr')) - ['attrs', 'meta'] - -Inspect the array metadata:: - - >>> print(open('example.zarr/meta').read()) - { - "chunks": [ - 10, - 10 - ], - "compression": "zlib", - "compression_opts": 1, - "dtype": ">> print(open('example.zarr/attrs').read()) - {} - -Set some data:: - - >>> z = zarr.Array(store) - >>> z[0:10, 0:10] = 1 - >>> sorted(os.listdir('example.zarr')) - ['0.0', 'attrs', 'meta'] - -Set some more data:: - - >>> z[0:10, 10:20] = 2 - >>> z[10:20, :] = 3 - >>> sorted(os.listdir('example.zarr')) - ['0.0', '0.1', '1.0', '1.1', 'attrs', 'meta'] - -Manually decompress a single chunk for illustration:: - - >>> import zlib - >>> b = zlib.decompress(open('example.zarr/0.0', 'rb').read()) - >>> import numpy as np - >>> a = np.frombuffer(b, dtype='>> a - array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32) - -Modify the array attributes:: - - >>> z.attrs['foo'] = 42 - >>> z.attrs['bar'] = 'apples' - >>> z.attrs['baz'] = [1, 2, 3, 4] - >>> print(open('example.zarr/attrs').read()) - { - "bar": "apples", - "baz": [ - 1, - 2, - 3, - 4 - ], - "foo": 42 - } +The V1 Specification has been migrated to its website → +https://zarr-specs.readthedocs.io/. diff --git a/docs/spec/v2.rst b/docs/spec/v2.rst index c1e12e121..deb6d46ce 100644 --- a/docs/spec/v2.rst +++ b/docs/spec/v2.rst @@ -3,563 +3,5 @@ Zarr Storage Specification Version 2 ==================================== -This document provides a technical specification of the protocol and format -used for storing Zarr arrays. The key words "MUST", "MUST NOT", "REQUIRED", -"SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and -"OPTIONAL" in this document are to be interpreted as described in `RFC 2119 -`_. - -Status ------- - -This specification is the latest version. See :ref:`spec` for previous -versions. - -.. _spec_v2_storage: - -Storage -------- - -A Zarr array can be stored in any storage system that provides a key/value -interface, where a key is an ASCII string and a value is an arbitrary sequence -of bytes, and the supported operations are read (get the sequence of bytes -associated with a given key), write (set the sequence of bytes associated with -a given key) and delete (remove a key/value pair). - -For example, a directory in a file system can provide this interface, where -keys are file names, values are file contents, and files can be read, written -or deleted via the operating system. Equally, an S3 bucket can provide this -interface, where keys are resource names, values are resource contents, and -resources can be read, written or deleted via HTTP. - -Below an "array store" refers to any system implementing this interface. - -.. _spec_v2_array: - -Arrays ------- - -.. _spec_v2_array_metadata: - -Metadata -~~~~~~~~ - -Each array requires essential configuration metadata to be stored, enabling -correct interpretation of the stored data. This metadata is encoded using JSON -and stored as the value of the ".zarray" key within an array store. - -The metadata resource is a JSON object. The following keys MUST be present -within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. -shape - A list of integers defining the length of each dimension of the array. -chunks - A list of integers defining the length of each dimension of a chunk of the - array. Note that all chunks within a Zarr array have the same shape. -dtype - A string or list defining a valid data type for the array. See also - the subsection below on data type encoding. -compressor - A JSON object identifying the primary compression codec and providing - configuration parameters, or ``null`` if no compressor is to be used. - The object MUST contain an ``"id"`` key identifying the codec to be used. -fill_value - A scalar value providing the default value to use for uninitialized - portions of the array, or ``null`` if no fill_value is to be used. -order - Either "C" or "F", defining the layout of bytes within each chunk of the - array. "C" means row-major order, i.e., the last dimension varies fastest; - "F" means column-major order, i.e., the first dimension varies fastest. -filters - A list of JSON objects providing codec configurations, or ``null`` if no - filters are to be applied. Each codec configuration object MUST contain a - ``"id"`` key identifying the codec to be used. - -The following keys MAY be present within the object: - -dimension_separator - If present, either the string ``"."`` or ``"/"`` defining the separator placed - between the dimensions of a chunk. If the value is not set, then the - default MUST be assumed to be ``"."``, leading to chunk keys of the form "0.0". - Arrays defined with ``"/"`` as the dimension separator can be considered to have - nested, or hierarchical, keys of the form "0/0" that SHOULD where possible - produce a directory-like structure. - -Other keys SHOULD NOT be present within the metadata object and SHOULD be -ignored by implementations. - -For example, the JSON object below defines a 2-dimensional array of 64-bit -little-endian floating point numbers with 10000 rows and 10000 columns, divided -into chunks of 1000 rows and 1000 columns (so there will be 100 chunks in total -arranged in a 10 by 10 grid). Within each chunk the data are laid out in C -contiguous order. Each chunk is encoded using a delta filter and compressed -using the Blosc compression library prior to storage:: - - { - "chunks": [ - 1000, - 1000 - ], - "compressor": { - "id": "blosc", - "cname": "lz4", - "clevel": 5, - "shuffle": 1 - }, - "dtype": "`. The format -consists of 3 parts: - -* One character describing the byteorder of the data (``"<"``: little-endian; - ``">"``: big-endian; ``"|"``: not-relevant) -* One character code giving the basic type of the array (``"b"``: Boolean (integer - type where all values are only True or False); ``"i"``: integer; ``"u"``: unsigned - integer; ``"f"``: floating point; ``"c"``: complex floating point; ``"m"``: timedelta; - ``"M"``: datetime; ``"S"``: string (fixed-length sequence of char); ``"U"``: unicode - (fixed-length sequence of Py_UNICODE); ``"V"``: other (void * – each item is a - fixed-size chunk of memory)) -* An integer specifying the number of bytes the type uses. - -The byte order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and -``"|S12"`` are valid data type encodings. - -For datetime64 ("M") and timedelta64 ("m") data types, these MUST also include the -units within square brackets. A list of valid units and their definitions are given in -the :ref:`NumPy documentation on Datetimes and Timedeltas -`. -For example, ``"`. Each -sub-list has the form ``[fieldname, datatype, shape]`` where ``shape`` -is optional. ``fieldname`` is a string, ``datatype`` is a string -specifying a simple data type (see above), and ``shape`` is a list of -integers specifying subarray shape. For example, the JSON list below -defines a data type composed of three single-byte unsigned integer -fields named "r", "g" and "b":: - - [["r", "|u1"], ["g", "|u1"], ["b", "|u1"]] - -For example, the JSON list below defines a data type composed of three -fields named "x", "y" and "z", where "x" and "y" each contain 32-bit -floats, and each item in "z" is a 2 by 2 array of floats:: - - [["x", "`_ -produces a sequence of bytes that begins with a 16-byte header followed by -compressed data. - -The compressed sequence of bytes for each chunk is stored under a key formed -from the index of the chunk within the grid of chunks representing the array. -To form a string key for a chunk, the indices are converted to strings and -concatenated with the period character (".") separating each index. For -example, given an array with shape (10000, 10000) and chunk shape (1000, 1000) -there will be 100 chunks laid out in a 10 by 10 grid. The chunk with indices -(0, 0) provides data for rows 0-999 and columns 0-999 and is stored under the -key "0.0"; the chunk with indices (2, 4) provides data for rows 2000-2999 and -columns 4000-4999 and is stored under the key "2.4"; etc. - -There is no need for all chunks to be present within an array store. If a chunk -is not present then it is considered to be in an uninitialized state. An -uninitialized chunk MUST be treated as if it was uniformly filled with the value -of the "fill_value" field in the array metadata. If the "fill_value" field is -``null`` then the contents of the chunk are undefined. - -Note that all chunks in an array have the same shape. If the length of any -array dimension is not exactly divisible by the length of the corresponding -chunk dimension then some chunks will overhang the edge of the array. The -contents of any chunk region falling outside the array are undefined. - -.. _spec_v2_array_filters: - -Filters -~~~~~~~ - -Optionally a sequence of one or more filters can be used to transform chunk -data prior to compression. When storing data, filters are applied in the order -specified in array metadata to encode data, then the encoded data are passed to -the primary compressor. When retrieving data, stored chunk data are -decompressed by the primary compressor then decoded using filters in the -reverse order. - -.. _spec_v2_hierarchy: - -Hierarchies ------------ - -.. _spec_v2_hierarchy_paths: - -Logical storage paths -~~~~~~~~~~~~~~~~~~~~~ - -Multiple arrays can be stored in the same array store by associating each array -with a different logical path. A logical path is simply an ASCII string. The -logical path is used to form a prefix for keys used by the array. For example, -if an array is stored at logical path "foo/bar" then the array metadata will be -stored under the key "foo/bar/.zarray", the user-defined attributes will be -stored under the key "foo/bar/.zattrs", and the chunks will be stored under -keys like "foo/bar/0.0", "foo/bar/0.1", etc. - -To ensure consistent behaviour across different storage systems, logical paths -MUST be normalized as follows: - -* Replace all backward slash characters ("\\\\") with forward slash characters - ("/") -* Strip any leading "/" characters -* Strip any trailing "/" characters -* Collapse any sequence of more than one "/" character into a single "/" - character - -The key prefix is then obtained by appending a single "/" character to the -normalized logical path. - -After normalization, if splitting a logical path by the "/" character results -in any path segment equal to the string "." or the string ".." then an error -MUST be raised. - -N.B., how the underlying array store processes requests to store values under -keys containing the "/" character is entirely up to the store implementation -and is not constrained by this specification. E.g., an array store could simply -treat all keys as opaque ASCII strings; equally, an array store could map -logical paths onto some kind of hierarchical storage (e.g., directories on a -file system). - -.. _spec_v2_hierarchy_groups: - -Groups -~~~~~~ - -Arrays can be organized into groups which can also contain other groups. A -group is created by storing group metadata under the ".zgroup" key under some -logical path. E.g., a group exists at the root of an array store if the -".zgroup" key exists in the store, and a group exists at logical path "foo/bar" -if the "foo/bar/.zgroup" key exists in the store. - -If the user requests a group to be created under some logical path, then groups -MUST also be created at all ancestor paths. E.g., if the user requests group -creation at path "foo/bar" then groups MUST be created at path "foo" and the -root of the store, if they don't already exist. - -If the user requests an array to be created under some logical path, then -groups MUST also be created at all ancestor paths. E.g., if the user requests -array creation at path "foo/bar/baz" then groups must be created at path -"foo/bar", path "foo", and the root of the store, if they don't already exist. - -The group metadata resource is a JSON object. The following keys MUST be present -within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. - -Other keys MUST NOT be present within the metadata object. - -The members of a group are arrays and groups stored under logical paths that -are direct children of the parent group's logical path. E.g., if groups exist -under the logical paths "foo" and "foo/bar" and an array exists at logical path -"foo/baz" then the members of the group at path "foo" are the group at path -"foo/bar" and the array at path "foo/baz". - -.. _spec_v2_attrs: - -Attributes ----------- - -An array or group can be associated with custom attributes, which are arbitrary -key/value pairs with application-specific meaning. Custom attributes are encoded -as a JSON object and stored under the ".zattrs" key within an array store. The -".zattrs" key does not have to be present, and if it is absent the attributes -should be treated as empty. - -For example, the JSON object below encodes three attributes named -"foo", "bar" and "baz":: - - { - "foo": 42, - "bar": "apples", - "baz": [1, 2, 3, 4] - } - -.. _spec_v2_examples: - -Examples --------- - -Storing a single array -~~~~~~~~~~~~~~~~~~~~~~ - -Below is an example of storing a Zarr array, using a directory on the -local file system as storage. - -Create an array:: - - >>> import zarr - >>> store = zarr.DirectoryStore('data/example.zarr') - >>> a = zarr.create(shape=(20, 20), chunks=(10, 10), dtype='i4', - ... fill_value=42, compressor=zarr.Zlib(level=1), - ... store=store, overwrite=True) - -No chunks are initialized yet, so only the ".zarray" and ".zattrs" keys -have been set in the store:: - - >>> import os - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray'] - -Inspect the array metadata:: - - >>> print(open('data/example.zarr/.zarray').read()) - { - "chunks": [ - 10, - 10 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": ">> a[0:10, 0:10] = 1 - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '0.0'] - -Set some more data:: - - >>> a[0:10, 10:20] = 2 - >>> a[10:20, :] = 3 - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '0.0', '0.1', '1.0', '1.1'] - -Manually decompress a single chunk for illustration:: - - >>> import zlib - >>> buf = zlib.decompress(open('data/example.zarr/0.0', 'rb').read()) - >>> import numpy as np - >>> chunk = np.frombuffer(buf, dtype='>> chunk - array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32) - -Modify the array attributes:: - - >>> a.attrs['foo'] = 42 - >>> a.attrs['bar'] = 'apples' - >>> a.attrs['baz'] = [1, 2, 3, 4] - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1'] - >>> print(open('data/example.zarr/.zattrs').read()) - { - "bar": "apples", - "baz": [ - 1, - 2, - 3, - 4 - ], - "foo": 42 - } - -Storing multiple arrays in a hierarchy -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Below is an example of storing multiple Zarr arrays organized into a group -hierarchy, using a directory on the local file system as storage. This storage -implementation maps logical paths onto directory paths on the file system, -however this is an implementation choice and is not required. - -Setup the store:: - - >>> import zarr - >>> store = zarr.DirectoryStore('data/group.zarr') - -Create the root group:: - - >>> root_grp = zarr.group(store, overwrite=True) - -The metadata resource for the root group has been created:: - - >>> import os - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup'] - -Inspect the group metadata:: - - >>> print(open('data/group.zarr/.zgroup').read()) - { - "zarr_format": 2 - } - -Create a sub-group:: - - >>> sub_grp = root_grp.create_group('foo') - -What has been stored:: - - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup', 'foo'] - >>> sorted(os.listdir('data/group.zarr/foo')) - ['.zgroup'] - -Create an array within the sub-group:: - - >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10)) - >>> a[:] = 42 - -Set a custom attributes:: - - >>> a.attrs['comment'] = 'answer to life, the universe and everything' - -What has been stored:: - - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup', 'foo'] - >>> sorted(os.listdir('data/group.zarr/foo')) - ['.zgroup', 'bar'] - >>> sorted(os.listdir('data/group.zarr/foo/bar')) - ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1'] - -Here is the same example using a Zip file as storage:: - - >>> store = zarr.ZipStore('data/group.zip', mode='w') - >>> root_grp = zarr.group(store) - >>> sub_grp = root_grp.create_group('foo') - >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10)) - >>> a[:] = 42 - >>> a.attrs['comment'] = 'answer to life, the universe and everything' - >>> store.close() - -What has been stored:: - - >>> import zipfile - >>> zf = zipfile.ZipFile('data/group.zip', mode='r') - >>> for name in sorted(zf.namelist()): - ... print(name) - .zgroup - foo/.zgroup - foo/bar/.zarray - foo/bar/.zattrs - foo/bar/0.0 - foo/bar/0.1 - foo/bar/1.0 - foo/bar/1.1 - -.. _spec_v2_changes: - -Changes -------- - -Version 2 clarifications -~~~~~~~~~~~~~~~~~~~~~~~~ - -The following changes have been made to the version 2 specification since it was -initially published to clarify ambiguities and add some missing information. - -* The specification now describes how bytes fill values should be encoded and - decoded for arrays with a fixed-length byte string data type (:issue:`165`, - :issue:`176`). - -* The specification now clarifies that units must be specified for datetime64 and - timedelta64 data types (:issue:`85`, :issue:`215`). - -* The specification now clarifies that the '.zattrs' key does not have to be present for - either arrays or groups, and if absent then custom attributes should be treated as - empty. - -* The specification now describes how structured datatypes with - subarray shapes and/or with nested structured data types are encoded - in array metadata (:issue:`111`, :issue:`296`). - -* Clarified the key/value pairs of custom attributes as "arbitrary" rather than - "simple". - -Changes from version 1 to version 2 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following changes were made between version 1 and version 2 of this specification: - -* Added support for storing multiple arrays in the same store and organising - arrays into hierarchies using groups. -* Array metadata is now stored under the ".zarray" key instead of the "meta" - key. -* Custom attributes are now stored under the ".zattrs" key instead of the - "attrs" key. -* Added support for filters. -* Changed encoding of "fill_value" field within array metadata. -* Changed encoding of compressor information within array metadata to be - consistent with representation of filter information. +The V2 Specification has been migrated to its website → +https://zarr-specs.readthedocs.io/. diff --git a/docs/spec/v3.rst b/docs/spec/v3.rst index bd8852707..3d39f35ba 100644 --- a/docs/spec/v3.rst +++ b/docs/spec/v3.rst @@ -1,7 +1,7 @@ .. _spec_v3: Zarr Storage Specification Version 3 -======================================================= +==================================== The V3 Specification has been migrated to its website → https://zarr-specs.readthedocs.io/. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index a40422490..619392a17 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -774,7 +774,7 @@ the following code:: Any other compatible storage class could be used in place of :class:`zarr.storage.DirectoryStore` in the code examples above. For example, -here is an array stored directly into a Zip file, via the +here is an array stored directly into a ZIP archive, via the :class:`zarr.storage.ZipStore` class:: >>> store = zarr.ZipStore('data/example.zip', mode='w') @@ -798,12 +798,12 @@ Re-open and check that data have been written:: [42, 42, 42, ..., 42, 42, 42]], dtype=int32) >>> store.close() -Note that there are some limitations on how Zip files can be used, because items -within a Zip file cannot be updated in place. This means that data in the array +Note that there are some limitations on how ZIP archives can be used, because items +within a ZIP archive cannot be updated in place. This means that data in the array should only be written once and write operations should be aligned with chunk boundaries. Note also that the ``close()`` method must be called after writing any data to the store, otherwise essential records will not be written to the -underlying zip file. +underlying ZIP archive. Another storage alternative is the :class:`zarr.storage.DBMStore` class, added in Zarr version 2.2. This class allows any DBM-style database to be used for @@ -846,7 +846,7 @@ respectively require the `redis-py `_ and `pymongo `_ packages to be installed. For compatibility with the `N5 `_ data format, Zarr also provides -an N5 backend (this is currently an experimental feature). Similar to the zip storage class, an +an N5 backend (this is currently an experimental feature). Similar to the ZIP storage class, an :class:`zarr.n5.N5Store` can be instantiated directly:: >>> store = zarr.N5Store('data/example.n5') @@ -868,7 +868,7 @@ implementations of the ``MutableMapping`` interface for Amazon S3 (`S3Map Distributed File System (`HDFSMap `_) and Google Cloud Storage (`GCSMap -`_), which +`_), which can be used with Zarr. Here is an example using S3Map to read an array created previously:: @@ -1000,6 +1000,32 @@ separately from Zarr. .. _tutorial_copy: +Accessing ZIP archives on S3 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The built-in :class:`zarr.storage.ZipStore` will only work with paths on the local file-system; however +it is possible to access ZIP-archived Zarr data on the cloud via the `ZipFileSystem `_ +class from ``fsspec``. The following example demonstrates how to access +a ZIP-archived Zarr group on s3 using `s3fs `_ and ``ZipFileSystem``: + + >>> s3_path = "s3://path/to/my.zarr.zip" + >>> + >>> s3 = s3fs.S3FileSystem() + >>> f = s3.open(s3_path) + >>> fs = ZipFileSystem(f, mode="r") + >>> store = FSMap("", fs, check=False) + >>> + >>> # caching may improve performance when repeatedly reading the same data + >>> cache = zarr.storage.LRUStoreCache(store, max_size=2**28) + >>> z = zarr.group(store=cache) + +This store can also be generated with ``fsspec``'s handler chaining, like so: + + >>> store = zarr.storage.FSStore(url=f"zip::{s3_path}", mode="r") + +This can be especially useful if you have a very large ZIP-archived Zarr array or group on s3 +and only need to access a small portion of it. + Consolidating metadata ~~~~~~~~~~~~~~~~~~~~~~ @@ -1136,7 +1162,7 @@ re-compression, and so should be faster. E.g.:: └── spam (100,) int64 >>> new_root['foo/bar/baz'][:] array([ 0, 1, 2, ..., 97, 98, 99]) - >>> store2.close() # zip stores need to be closed + >>> store2.close() # ZIP stores need to be closed .. _tutorial_strings: diff --git a/pyproject.toml b/pyproject.toml index b77260b00..2bc2b526e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,6 +109,7 @@ Homepage = "https://github.com/zarr-developers/zarr-python" exclude_lines = [ "pragma: no cover", "pragma: ${PY_MAJOR_VERSION} no cover", + '.*\.\.\.' # Ignore "..." lines ] [tool.coverage.run] diff --git a/v3-roadmap-and-design.md b/v3-roadmap-and-design.md new file mode 100644 index 000000000..696799e56 --- /dev/null +++ b/v3-roadmap-and-design.md @@ -0,0 +1,429 @@ +# Zarr Python Roadmap + +- Status: draft +- Author: Joe Hamman +- Created On: October 31, 2023 +- Input from: + - Davis Bennett / @d-v-b + - Norman Rzepka / @normanrz + - Deepak Cherian @dcherian + - Brian Davis / @monodeldiablo + - Oliver McCormack / @olimcc + - Ryan Abernathey / @rabernat + - Jack Kelly / @JackKelly + - Martin Durrant / @martindurant + +## Introduction + +This document lays out a design proposal for version 3.0 of the [Zarr-Python](https://zarr.readthedocs.io/en/stable/) package. A specific focus of the design is to bring Zarr-Python's API up to date with the [Zarr V3 specification](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html), with the hope of enabling the development of the many features and extensions that motivated the V3 Spec. The ideas presented here are expected to result in a major release of Zarr-Python (version 3.0) including significant a number of breaking API changes. +For clarity, “V3” will be used to describe the version of the Zarr specification and “3.0” will be used to describe the release tag of the Zarr-Python project. + +### Current status of V3 in Zarr-Python + +During the development of the V3 Specification, a [prototype implementation](https://github.com/zarr-developers/zarr-python/pull/898) was added to the Zarr-Python library. Since that implementation, the V3 spec evolved in significant ways and as a result, the Zarr-Python library is now out of sync with the approved spec. Downstream libraries (e.g. [Xarray](https://github.com/pydata/xarray)) have added support for this implementation and will need to migrate to the accepted spec when its available in Zarr-Python. + +## Goals + +- Provide a complete implementation of Zarr V3 through the Zarr-Python API +- Clear the way for exciting extensions / ZEPs (i.e. [sharding](https://zarr-specs.readthedocs.io/en/latest/v3/codecs/sharding-indexed/v1.0.html), [variable chunking](https://zarr.dev/zeps/draft/ZEP0003.html), etc.) +- Provide a developer API that can be used to implement and register V3 extensions +- Improve the performance of Zarr-Python by streamlining the interface between the Store layer and higher level APIs (e.g. Groups and Arrays) +- Clean up the internal and user facing APIs +- Improve code quality and robustness (e.g. achieve 100% type hint coverage) +- Align the Zarr-Python array API with the [array API Standard](https://data-apis.org/array-api/latest/) + +## Examples of what 3.0 will enable? +1. Reading and writing V3 spec-compliant groups and arrays +2. V3 extensions including sharding and variable chunking. +3. Improved performance by leveraging concurrency when creating/reading/writing to stores (imagine a `create_hierarchy(zarr_objects)` function). +4. User-developed extensions (e.g. storage-transformers) can be registered with Zarr-Python at runtime + +## Non-goals (of this document) + +- Implementation of any unaccepted Zarr V3 extensions +- Major revisions to the Zarr V3 spec + +## Requirements + +1. Read and write spec compliant V2 and V3 data +2. Limit unnecessary traffic to/from the store +3. Cleanly define the Array/Group/Store abstractions +4. Cleanly define how V2 will be supported going forward +5. Provide a clear roadmap to help users upgrade to 3.0 +6. Developer tools / hooks for registering extensions + +## Design + +### Async API + +Zarr-Python is an IO library. As such, supporting concurrent action against the storage layer is critical to achieving acceptable performance. The Zarr-Python 2 was not designed with asynchronous computation in mind and as a result has struggled to effectively leverage the benefits of concurrency. At one point, `getitems` and `setitems` support was added to the Zarr store model but that is only used for operating on a set of chunks in a single variable. + +With Zarr-Python 3.0, we have the opportunity to revisit this design. The proposal here is as follows: + +1. The `Store` interface will be entirely async. +2. On top of the async `Store` interface, we will provide an `AsyncArray` and `AsyncGroup` interface. +3. Finally, the primary user facing API will be synchronous `Array` and `Group` classes that wrap the async equivalents. + +**Examples** + +- **Store** + + ```python + class Store: + ... + async def get(self, key: str) -> bytes: + ... + async def get_partial_values(self, key_ranges: List[Tuple[str, Tuple[int, Optional[int]]]]) -> bytes: + ... + # (no sync interface here) + ``` +- **Array** + + ```python + class AsyncArray: + ... + + async def getitem(self, selection: Selection) -> np.ndarray: + # the core logic for getitem goes here + + class Array: + _async_array: AsyncArray + + def __getitem__(self, selection: Selection) -> np.ndarray: + return sync(self._async_array.getitem(selection)) + ``` +- **Group** + + ```python + class AsyncGroup: + ... + + async def create_group(self, path: str, **kwargs) -> AsyncGroup: + # the core logic for create_group goes here + + class Group: + _async_group: AsyncGroup + + def create_group(self, path: str, **kwargs) -> Group: + return sync(self._async_group.create_group(path, **kwargs)) + ``` +**Internal Synchronization API** + +With the `Store` and core `AsyncArray`/ `AsyncGroup` classes being predominantly async, Zarr-Python will need an internal API to provide a synchronous API. The proposal here is to use the approach in [fsspec](https://github.com/fsspec/filesystem_spec/blob/master/fsspec/asyn.py) to provide a high-level `sync` function that takes an `awaitable` and runs it in its managed IO Loop / thread. + +**FAQ** +1. Why two levels of Arrays/groups? + a. First, this is an intentional decision and departure from the current Zarrita implementation + b. The idea is that users rarely want to mix interfaces. Either they are working within an async context (currently quite rare) or they are in a typical synchronous context. + c. Splitting the two will allow us to clearly define behavior on the `AsyncObj` and simply wrap it in the `SyncObj`. +2. What if a store is only has a synchronous backend? + a. First off, this is expected to be a fairly rare occurrence. Most storage backends have async interfaces. + b. But in the event a storage backend doesn’t have a async interface, there is nothing wrong with putting synchronous code in `async` methods. There are approaches to enabling concurrent action through wrappers like AsyncIO's `loop.run_in_executor` ([ref 1](https://stackoverflow.com/questions/38865050/is-await-in-python3-cooperative-multitasking ), [ref 2](https://stackoverflow.com/a/43263397/732596), [ref 3](https://bbc.github.io/cloudfit-public-docs/asyncio/asyncio-part-5.html), [ref 4](https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.run_in_executor). +3. Will Zarr help manage the async contexts encouraged by some libraries (e.g. [AioBotoCore](https://aiobotocore.readthedocs.io/en/latest/tutorial.html#using-botocore))? + a. Many async IO libraries require entering an async context before interacting with the API. We expect some experimentation to be needed here but the initial design will follow something close to what fsspec does ([example in s3fs](https://github.com/fsspec/s3fs/blob/949442693ec940b35cda3420c17a864fbe426567/s3fs/core.py#L527)). +4. Why not provide a synchronous Store interface? + a. We could but this design is simpler. It would mean supporting it in the `AsyncGroup` and `AsyncArray` classes which, may be more trouble than its worth. Storage backends that do not have an async API will be encouraged to wrap blocking calls in an async wrapper (e.g. `loop.run_in_executor`). + +### Store API + +The `Store` API is specified directly in the V3 specification. All V3 stores should implement this abstract API, omitting Write and List support as needed. As described above, all stores will be expected to expose the required methods as async methods. + +**Example** + +```python +class ReadWriteStore: + ... + async def get(self, key: str) -> bytes: + ... + + async def get_partial_values(self, key_ranges: List[Tuple[str, int, int]) -> bytes: + ... + + async def set(self, key: str, value: Union[bytes, bytearray, memoryview]) -> None: + ... # required for writable stores + + async def set_partial_values(self, key_start_values: List[Tuple[str, int, Union[bytes, bytearray, memoryview]]]) -> None: + ... # required for writable stores + + async def list(self) -> List[str]: + ... # required for listable stores + + async def list_prefix(self, prefix: str) -> List[str]: + ... # required for listable stores + + async def list_dir(self, prefix: str) -> List[str]: + ... # required for listable stores + + # additional (optional methods) + async def getsize(self, prefix: str) -> int: + ... + + async def rename(self, src: str, dest: str) -> None + ... + +``` + +Recognizing that there are many Zarr applications today that rely on the `MutableMapping` interface supported by Zarr-Python 2, a wrapper store will be developed to allow existing stores to plug directly into this API. + +### Array API + +The user facing array interface will implement a subset of the [Array API Standard](https://data-apis.org/array-api/latest/). Most of the computational parts of the Array API Standard don’t fit into Zarr right now. That’s okay. What matters most is that we ensure we can give downstream applications a compliant API. + +*Note, Zarr already does most of this so this is more about formalizing the relationship than a substantial change in API.* + +| | Included | Not Included | Unknown / Maybe possible? | +| --- | --- | --- | --- | +| Attributes | `dtype` | `mT` | `device` | +| | `ndim` | `T` | | +| | `shape` | | | +| | `size` | | | +| Methods | `__getitem__` | `__array_namespace__` | `to_device` | +| | `__setitem__` | `__abs__` | `__bool__` | +| | `__eq__` | `__add__` | `__complex__` | +| | `__bool__` | `__and__` | `__dlpack__` | +| | | `__floordiv__` | `__dlpack_device__` | +| | | `__ge__` | `__float__` | +| | | `__gt__` | `__index__` | +| | | `__invert__` | `__int__` | +| | | `__le__` | | +| | | `__lshift__` | | +| | | `__lt__` | | +| | | `__matmul__` | | +| | | `__mod__` | | +| | | `__mul__` | | +| | | `__ne__` | | +| | | `__neg__` | | +| | | `__or__` | | +| | | `__pos__` | | +| | | `__pow__` | | +| | | `__rshift__` | | +| | | `__sub__` | | +| | | `__truediv__` | | +| | | `__xor__` | | +| Creation functions (`zarr.creation`) | `zeros` | | `arange` | +| | `zeros_like` | | `asarray` | +| | `ones` | | `eye` | +| | `ones_like` | | `from_dlpack` | +| | `full` | | `linspace` | +| | `full_like` | | `meshgrid` | +| | `empty` | | `tril` | +| | `empty_like` | | `triu` | + +In addition to the core array API defined above, the Array class should have the following Zarr specific properties: + +- `.metadata` (see Metadata Interface below) +- `.attrs` - (pull from metadata object) +- `.info` - (pull from existing property †) + +*† In Zarr-Python 2, the info property lists the store to identify initialized chunks. By default this will be turned off in 3.0 but will be configurable.* + +**Indexing** + +Zarr-Python currently supports `__getitem__` style indexing and the special `oindex` and `vindex` indexers. These are not part of the current Array API standard (see [data-apis/array-api\#669](https://github.com/data-apis/array-api/issues/669)) but they have been [proposed as a NEP](https://numpy.org/neps/nep-0021-advanced-indexing.html). Zarr-Python will maintain these in 3.0. + +We are also exploring a new high-level indexing API that will enabled optimized batch/concurrent loading of many chunks. We expect this to be important to enable performant loading of data in the context of sharding. See [this discussion](https://github.com/zarr-developers/zarr-python/discussions/1569) for more detail. + +Concurrent indexing across multiple arrays will be possible using the AsyncArray API. + +**Async and Sync Array APIs** + +Most the logic to support Zarr Arrays will live in the `AsyncArray` class. There are a few notable differences that should be called out. + +| Sync Method | Async Method | +| --- | --- | +| `__getitem__` | `getitem` | +| `__setitem__` | `setitem` | +| `__eq__` | `equals` | + +**Metadata interface** + +Zarr-Python 2.* closely mirrors the V2 spec metadata schema in the Array and Group classes. In 3.0, we plan to move the underlying metadata representation to a separate interface (e.g. `Array.metadata`). This interface will return either a `V2ArrayMetadata` or `V3ArrayMetadata` object (both will inherit from a parent `ArrayMetadataABC` class. The `V2ArrayMetadata` and `V3ArrayMetadata` classes will be responsible for producing valid JSON representations of their metadata, and yielding a consistent view to the `Array` or `Group` class. + +### Group API + +The main question is how closely we should follow the existing Zarr-Python implementation / `MutableMapping` interface. The table below shows the primary `Group` methods in Zarr-Python 2 and attempts to identify if and how they would be implemented in 3.0. + +| V2 Group Methods | `AsyncGroup` | `Group` | `h5py_compat.Group`` | +| --- | --- | --- | --- | +| `__len__` | `length` | `__len__` | `__len__` | +| `__iter__` | `__aiter__` | `__iter__` | `__iter__` | +| `__contains__` | `contains` | `__contains__` | `__contains__` | +| `__getitem__` | `getitem` | `__getitem__` | `__getitem__` | +| `__enter__` | N/A | N/A | `__enter__` | +| `__exit__` | N/A | N/A | `__exit__` | +| `group_keys` | `group_keys` | `group_keys` | N/A | +| `groups` | `groups` | `groups` | N/A | +| `array_keys` | `array_key` | `array_keys` | N/A | +| `arrays` | `arrays`* | `arrays` | N/A | +| `visit` | ? | ? | `visit` | +| `visitkeys` | ? | ? | ? | +| `visitvalues` | ? | ? | ? | +| `visititems` | ? | ? | `visititems` | +| `tree` | `tree` | `tree` | `Both` | +| `create_group` | `create_group` | `create_group` | `create_group` | +| `require_group` | N/A | N/A | `require_group` | +| `create_groups` | ? | ? | N/A | +| `require_groups` | ? | ? | ? | +| `create_dataset` | N/A | N/A | `create_dataset` | +| `require_dataset` | N/A | N/A | `require_dataset` | +| `create` | `create_array` | `create_array` | N/A | +| `empty` | `empty` | `empty` | N/A | +| `zeros` | `zeros` | `zeros` | N/A | +| `ones` | `ones` | `ones` | N/A | +| `full` | `full` | `full` | N/A | +| `array` | `create_array` | `create_array` | N/A | +| `empty_like` | `empty_like` | `empty_like` | N/A | +| `zeros_like` | `zeros_like` | `zeros_like` | N/A | +| `ones_like` | `ones_like` | `ones_like` | N/A | +| `full_like` | `full_like` | `full_like` | N/A | +| `move` | `move` | `move` | `move` | + +**`zarr.h5compat.Group`** + +Zarr-Python 2.* made an attempt to align its API with that of [h5py](https://docs.h5py.org/en/stable/index.html). With 3.0, we will relax this alignment in favor of providing an explicit compatibility module (`zarr.h5py_compat`). This module will expose the `Group` and `Dataset` APIs that map to Zarr-Python’s `Group` and `Array` objects. + +### Creation API + +Zarr-Python 2.* bundles together the creation and serialization of Zarr objects. Zarr-Python 3.* will make it possible to create objects in memory separate from serializing them. This will specifically enable writing hierarchies of Zarr objects in a single batch step. For example: + +```python + +arr1 = Array(shape=(10, 10), path="foo/bar", dtype="i4", store=store) +arr2 = Array(shape=(10, 10), path="foo/spam", dtype="f8", store=store) + +arr1.save() +arr2.save() + +# or equivalently + +zarr.save_many([arr1 ,arr2]) +``` + +*Note: this batch creation API likely needs additional design effort prior to implementation.* + +### Plugin API + +Zarr V3 was designed to be extensible at multiple layers. Zarr-Python will support these extensions through a combination of [Abstract Base Classes](https://docs.python.org/3/library/abc.html) (ABCs) and [Entrypoints](https://packaging.python.org/en/latest/specifications/entry-points/). + +**ABCs** + +Zarr V3 will expose Abstract base classes for the following objects: + +- `Store`, `ReadStore`, `ReadWriteStore`, `ReadListStore`, and `ReadWriteListStore` +- `BaseArray`, `SynchronousArray`, and `AsynchronousArray` +- `BaseGroup`, `SynchronousGroup`, and `AsynchronousGroup` +- `Codec`, `ArrayArrayCodec`, `ArrayBytesCodec`, `BytesBytesCodec` + +**Entrypoints** + +Lots more thinking here but the idea here is to provide entrypoints for `data type`, `chunk grid`, `chunk key encoding`, `codecs`, `storage_transformers` and `stores`. These might look something like: + +``` +entry_points=""" + [zarr.codecs] + blosc_codec=codec_plugin:make_blosc_codec + zlib_codec=codec_plugin:make_zlib_codec +""" +``` + +### Python type hints and static analysis + +Target 100% Mypy coverage in 3.0 source. + +### Observability + +A persistent problem in Zarr-Python is diagnosing problems that span many parts of the stack. To address this in 3.0, we will add a basic logging framework that can be used to debug behavior at various levels of the stack. We propose to add the separate loggers for the following namespaces: + +- `array` +- `group` +- `store` +- `codec` + +These should be documented such that users know how to activate them and developers know how to use them when developing extensions. + +### Dependencies + +Today, Zarr-Python has the following required dependencies: + +```python +dependencies = [ + 'asciitree', + 'numpy>=1.20,!=1.21.0', + 'fasteners', + 'numcodecs>=0.10.0', +] +``` + +What other dependencies should be considered? + +1. Attrs - Zarrita makes extensive use of the Attrs library +2. Fsspec - Zarrita has a hard dependency on Fsspec. This could be easily relaxed though. + +## Breaking changes relative to Zarr-Python 2.* + +1. H5py compat moved to a stand alone module? +2. `Group.__getitem__` support moved to `Group.members.__getitem__`? +3. Others? + +## Open questions + +1. How to treat V2 + a. Note: Zarrita currently implements a separate `V2Array` and `V3Array` classes. This feels less than ideal. + b. We could easily convert metadata from v2 to the V3 Array, but what about writing? + c. Ideally, we don’t have completely separate code paths. But if its too complicated to support both within one interface, its probably better. +2. How and when to remove the current implementation of V3. + a. It's hidden behind a hard-to-use feature flag so we probably don't need to do anything. +4. How to model runtime configuration? +5. Which extensions belong in Zarr-Python and which belong in separate packages? + a. We don't need to take a strong position on this here. It's likely that someone will want to put Sharding in. That will be useful to develop in parallel because it will give us a good test case for the plugin interface. + +## Testing + +Zarr-python 3.0 adds a major new dimension to Zarr: Async support. This also comes with a compatibility risk, we will need to thoroughly test support in key execution environments. Testing plan: +- Reuse the existing test suite for testing the `v3` API. + - `xfail` tests that expose breaking changes with `3.0 - breaking change` description. This will help identify additional and/or unintentional breaking changes + - Rework tests that were only testing internal APIs. +- Add a set of functional / integration tests targeting real-world workflows in various contexts (e.g. w/ Dask) + +## Development process + +Zarr-Python 3.0 will introduce a number of new APIs and breaking changes to existing APIs. In order to facilitate ongoing support for Zarr-Python 2.*, we will take on the following development process: + +- Create a `v3` branch that can be use for developing the core functionality apart from the `main` branch. This will allow us to support ongoing work and bug fixes on the `main` branch. +- Put the `3.0` APIs inside a `zarr.v3` module. Imports from this namespace will all be new APIs that users can develop and test against once the `v3` branch is merged to `main`. +- Kickstart the process by pulling in the current state of `zarrita` - which has many of the features described in this design. +- Release a series of 2.* releases with the `v3` namespace +- When `v3` is complete, move contents of `v3` to the package root + +**Milestones** + +Below are a set of specific milestones leading toward the completion of this process. As work begins, we expect this list to grow in specificity. + +1. Port current version of Zarrita to Zarr-Python +2. Formalize Async interface by splitting `Array` and `Group` objects into Sync and Async versions +4. Implement "fancy" indexing operations on the `AsyncArray` +6. Implement an abstract base class for the `Store` interface and a wrapper `Store` to make use of existing `MutableMapping` stores. +7. Rework the existing unit test suite to use the `v3` namespace. +8. Develop a plugin interface for extensions +9. Develop a set of functional and integration tests +10. Work with downstream libraries (Xarray, Dask, etc.) to test new APIs + +## TODOs + +The following subjects are not covered in detail above but perhaps should be. Including them here so they are not forgotten. + +1. [Store] Should Zarr provide an API for caching objects after first read/list/etc. Read only stores? +2. [Array] buffer protocol support +3. [Array] `meta_array` support +4. [Extensions] Define how Zarr-Python will consume the various plugin types +5. [Misc] H5py compatibility requires a bit more work and a champion to drive it forward. +6. [Misc] Define `chunk_store` API in 3.0 +7. [Misc] Define `synchronizer` API in 3.0 + +## References + +1. [Zarr-Python repository](https://github.com/zarr-developers/zarr-python) +2. [Zarr core specification (version 3.0) — Zarr specs documentation](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#) +3. [Zarrita repository](https://github.com/scalableminds/zarrita) +4. [Async-Zarr](https://github.com/martindurant/async-zarr) +5. [Zarr-Python Discussion Topic](https://github.com/zarr-developers/zarr-python/discussions/1569) From 124640aef093ffdd6bf8d23e15d7559620d506df Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 11 Oct 2024 15:13:40 -0700 Subject: [PATCH 5/5] Revert "V3 main sync w/ merge (#2335)" (#2339) This reverts commit ce5f53e3482727b7133def725af24576057aa164. --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/releases.yml | 2 +- LICENSE.txt | 2 +- README.md | 6 +- TEAM.md | 29 -- docs/conf.py | 16 +- docs/index.rst | 4 +- docs/installation.rst | 5 - docs/release.rst | 33 -- docs/spec/v1.rst | 267 +++++++++++- docs/spec/v2.rst | 562 +++++++++++++++++++++++++- docs/spec/v3.rst | 2 +- docs/tutorial.rst | 40 +- pyproject.toml | 1 - v3-roadmap-and-design.md | 429 -------------------- 15 files changed, 851 insertions(+), 549 deletions(-) delete mode 100644 TEAM.md delete mode 100644 v3-roadmap-and-design.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 705cd31cb..2e11f1461 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -27,7 +27,7 @@ body: attributes: label: Python Version description: Version of Python interpreter - placeholder: 3.10, 3.11, 3.12 etc. + placeholder: 3.8.5, 3.9, 3.10, etc. validations: required: true - type: input diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index bab53958d..f694039ce 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -16,7 +16,7 @@ jobs: submodules: true fetch-depth: 0 - - uses: actions/setup-python@v5.2.0 + - uses: actions/setup-python@v5 name: Install Python with: python-version: '3.11' diff --git a/LICENSE.txt b/LICENSE.txt index a4de1c39d..850a0d877 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015-2024 Zarr Developers +Copyright (c) 2015-2023 Zarr Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index e379c9719..b035ffa59 100644 --- a/README.md +++ b/README.md @@ -70,10 +70,10 @@ - Zulip + Gitter - - + + diff --git a/TEAM.md b/TEAM.md deleted file mode 100644 index e6975d7c0..000000000 --- a/TEAM.md +++ /dev/null @@ -1,29 +0,0 @@ -## Active core-developers -- @joshmoore (Josh Moore) -- @jni (Juan Nunez-Iglesias) -- @rabernat (Ryan Abernathey) -- @jhamman (Joe Hamman) -- @d-v-b (Davis Bennett) -- @jakirkham (jakirkham) -- @martindurant (Martin Durant) -- @normanrz (Norman Rzepka) -- @dstansby (David Stansby) -- @dcherian (Deepak Cherian) -- @TomAugspurger (Tom Augspurger) - -## Emeritus core-developers -- @alimanfoo (Alistair Miles) -- @shoyer (Stephan Hoyer) -- @ryan-williams (Ryan Williams) -- @jrbourbeau (James Bourbeau) -- @mzjp2 (Zain Patel) -- @grlee77 (Gregory Lee) - -## Former core-developers -- @jeromekelleher (Jerome Kelleher) -- @tjcrone (Tim Crone) -- @funkey (Jan Funke) -- @shikharsg -- @Carreau (Matthias Bussonnier) -- @dazzag24 -- @WardF (Ward Fisher) diff --git a/docs/conf.py b/docs/conf.py index 0a328ac25..ec8509837 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,8 +19,11 @@ import sphinx.application -from importlib.metadata import version as get_version - +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. import zarr import sphinx @@ -76,11 +79,12 @@ # General information about the project. project = "zarr" -copyright = "2024, Zarr Developers" +copyright = "2023, Zarr Developers" author = "Zarr Developers" -version = get_version("zarr") -release = get_version("zarr") +version = zarr.__version__ +# The full version, including alpha/beta/rc tags. +release = zarr.__version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -201,7 +205,7 @@ def setup(app: sphinx.application.Sphinx) -> None: # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -html_sidebars = {"tutorial": []} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. diff --git a/docs/index.rst b/docs/index.rst index 6b90b5a77..8e1b0740c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,13 +19,13 @@ Zarr-Python **Version**: |version| -**Download documentation**: `PDF/Zipped HTML `_ +**Download documentation**: `PDF/Zipped HTML/EPUB `_ **Useful links**: `Installation `_ | `Source Repository `_ | `Issue Tracker `_ | -`Zulip Chat `_ +`Gitter `_ Zarr is a file storage format for chunked, compressed, N-dimensional arrays based on an open-source specification. diff --git a/docs/installation.rst b/docs/installation.rst index 86da6d103..3d4ac4107 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -6,11 +6,6 @@ Zarr depends on NumPy. It is generally best to `install NumPy appropriate for your operating system and Python distribution. Other dependencies should be installed automatically if using one of the installation methods below. -Note: Zarr has endorsed `Scientific-Python SPEC 0 `_ and now follows the version support window as outlined below: - -- Python: 36 months after initial release -- Core package dependencies (e.g. NumPy): 24 months after initial release - Install Zarr from PyPI:: $ pip install zarr diff --git a/docs/release.rst b/docs/release.rst index 0b6775c4a..5c3a43a14 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -303,39 +303,6 @@ Documentation * doc: update release notes for 3.0.0.alpha. By :user:`Joe Hamman ` :issue:`1959`. -.. _release_2.18.3: - -2.18.3 ------- - -Enhancements -~~~~~~~~~~~~ -* Added support for creating a copy of data when converting a `zarr.Array` - to a numpy array. - By :user:`David Stansby ` (:issue:`2106`) and - :user:`Joe Hamman ` (:issue:`2123`). - -Maintenance -~~~~~~~~~~~ -* Removed support for Python 3.9. - By :user:`David Stansby ` (:issue:`2074`). - -* Fix a regression when using orthogonal indexing with a scalar. - By :user:`Deepak Cherian ` :issue:`1931` - -* Added compatibility with NumPy 2.1. - By :user:`David Stansby ` - -* Bump minimum NumPy version to 1.24. - :user:`Joe Hamman ` (:issue:`2127`). - -Deprecations -~~~~~~~~~~~~ - -* Deprecate :class:`zarr.n5.N5Store` and :class:`zarr.n5.N5FSStore`. These - stores are slated to be removed in Zarr Python 3.0. - By :user:`Joe Hamman ` :issue:`2085`. - .. _release_2.18.2: 2.18.2 diff --git a/docs/spec/v1.rst b/docs/spec/v1.rst index 27a0490e0..13f68ef36 100644 --- a/docs/spec/v1.rst +++ b/docs/spec/v1.rst @@ -3,5 +3,268 @@ Zarr Storage Specification Version 1 ==================================== -The V1 Specification has been migrated to its website → -https://zarr-specs.readthedocs.io/. +This document provides a technical specification of the protocol and +format used for storing a Zarr array. The key words "MUST", "MUST +NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", +"RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be +interpreted as described in `RFC 2119 +`_. + +Status +------ + +This specification is deprecated. See :ref:`spec` for the latest version. + +Storage +------- + +A Zarr array can be stored in any storage system that provides a +key/value interface, where a key is an ASCII string and a value is an +arbitrary sequence of bytes, and the supported operations are read +(get the sequence of bytes associated with a given key), write (set +the sequence of bytes associated with a given key) and delete (remove +a key/value pair). + +For example, a directory in a file system can provide this interface, +where keys are file names, values are file contents, and files can be +read, written or deleted via the operating system. Equally, an S3 +bucket can provide this interface, where keys are resource names, +values are resource contents, and resources can be read, written or +deleted via HTTP. + +Below an "array store" refers to any system implementing this +interface. + +Metadata +-------- + +Each array requires essential configuration metadata to be stored, +enabling correct interpretation of the stored data. This metadata is +encoded using JSON and stored as the value of the 'meta' key within an +array store. + +The metadata resource is a JSON object. The following keys MUST be +present within the object: + +zarr_format + An integer defining the version of the storage specification to which the + array store adheres. +shape + A list of integers defining the length of each dimension of the array. +chunks + A list of integers defining the length of each dimension of a chunk of the + array. Note that all chunks within a Zarr array have the same shape. +dtype + A string or list defining a valid data type for the array. See also + the subsection below on data type encoding. +compression + A string identifying the primary compression library used to compress + each chunk of the array. +compression_opts + An integer, string or dictionary providing options to the primary + compression library. +fill_value + A scalar value providing the default value to use for uninitialized + portions of the array. +order + Either 'C' or 'F', defining the layout of bytes within each chunk of the + array. 'C' means row-major order, i.e., the last dimension varies fastest; + 'F' means column-major order, i.e., the first dimension varies fastest. + +Other keys MAY be present within the metadata object however they MUST +NOT alter the interpretation of the required fields defined above. + +For example, the JSON object below defines a 2-dimensional array of +64-bit little-endian floating point numbers with 10000 rows and 10000 +columns, divided into chunks of 1000 rows and 1000 columns (so there +will be 100 chunks in total arranged in a 10 by 10 grid). Within each +chunk the data are laid out in C contiguous order, and each chunk is +compressed using the Blosc compression library:: + + { + "chunks": [ + 1000, + 1000 + ], + "compression": "blosc", + "compression_opts": { + "clevel": 5, + "cname": "lz4", + "shuffle": 1 + }, + "dtype": "`_. The +format consists of 3 parts: a character describing the byteorder of +the data (``<``: little-endian, ``>``: big-endian, ``|``: +not-relevant), a character code giving the basic type of the array, +and an integer providing the number of bytes the type uses. The byte +order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and +``"|S12"`` are valid data types. + +Structure data types (i.e., with multiple named fields) are encoded as +a list of two-element lists, following `NumPy array protocol type +descriptions (descr) +`_. +For example, the JSON list ``[["r", "|u1"], ["g", "|u1"], ["b", +"|u1"]]`` defines a data type composed of three single-byte unsigned +integers labelled 'r', 'g' and 'b'. + +Chunks +------ + +Each chunk of the array is compressed by passing the raw bytes for the +chunk through the primary compression library to obtain a new sequence +of bytes comprising the compressed chunk data. No header is added to +the compressed bytes or any other modification made. The internal +structure of the compressed bytes will depend on which primary +compressor was used. For example, the `Blosc compressor +`_ +produces a sequence of bytes that begins with a 16-byte header +followed by compressed data. + +The compressed sequence of bytes for each chunk is stored under a key +formed from the index of the chunk within the grid of chunks +representing the array. To form a string key for a chunk, the indices +are converted to strings and concatenated with the period character +('.') separating each index. For example, given an array with shape +(10000, 10000) and chunk shape (1000, 1000) there will be 100 chunks +laid out in a 10 by 10 grid. The chunk with indices (0, 0) provides +data for rows 0-999 and columns 0-999 and is stored under the key +'0.0'; the chunk with indices (2, 4) provides data for rows 2000-2999 +and columns 4000-4999 and is stored under the key '2.4'; etc. + +There is no need for all chunks to be present within an array +store. If a chunk is not present then it is considered to be in an +uninitialized state. An uninitialized chunk MUST be treated as if it +was uniformly filled with the value of the 'fill_value' field in the +array metadata. If the 'fill_value' field is ``null`` then the +contents of the chunk are undefined. + +Note that all chunks in an array have the same shape. If the length of +any array dimension is not exactly divisible by the length of the +corresponding chunk dimension then some chunks will overhang the edge +of the array. The contents of any chunk region falling outside the +array are undefined. + +Attributes +---------- + +Each array can also be associated with custom attributes, which are +simple key/value items with application-specific meaning. Custom +attributes are encoded as a JSON object and stored under the 'attrs' +key within an array store. Even if the attributes are empty, the +'attrs' key MUST be present within an array store. + +For example, the JSON object below encodes three attributes named +'foo', 'bar' and 'baz':: + + { + "foo": 42, + "bar": "apples", + "baz": [1, 2, 3, 4] + } + +Example +------- + +Below is an example of storing a Zarr array, using a directory on the +local file system as storage. + +Initialize the store:: + + >>> import zarr + >>> store = zarr.DirectoryStore('example.zarr') + >>> zarr.init_store(store, shape=(20, 20), chunks=(10, 10), + ... dtype='i4', fill_value=42, compression='zlib', + ... compression_opts=1, overwrite=True) + +No chunks are initialized yet, so only the 'meta' and 'attrs' keys +have been set:: + + >>> import os + >>> sorted(os.listdir('example.zarr')) + ['attrs', 'meta'] + +Inspect the array metadata:: + + >>> print(open('example.zarr/meta').read()) + { + "chunks": [ + 10, + 10 + ], + "compression": "zlib", + "compression_opts": 1, + "dtype": ">> print(open('example.zarr/attrs').read()) + {} + +Set some data:: + + >>> z = zarr.Array(store) + >>> z[0:10, 0:10] = 1 + >>> sorted(os.listdir('example.zarr')) + ['0.0', 'attrs', 'meta'] + +Set some more data:: + + >>> z[0:10, 10:20] = 2 + >>> z[10:20, :] = 3 + >>> sorted(os.listdir('example.zarr')) + ['0.0', '0.1', '1.0', '1.1', 'attrs', 'meta'] + +Manually decompress a single chunk for illustration:: + + >>> import zlib + >>> b = zlib.decompress(open('example.zarr/0.0', 'rb').read()) + >>> import numpy as np + >>> a = np.frombuffer(b, dtype='>> a + array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32) + +Modify the array attributes:: + + >>> z.attrs['foo'] = 42 + >>> z.attrs['bar'] = 'apples' + >>> z.attrs['baz'] = [1, 2, 3, 4] + >>> print(open('example.zarr/attrs').read()) + { + "bar": "apples", + "baz": [ + 1, + 2, + 3, + 4 + ], + "foo": 42 + } diff --git a/docs/spec/v2.rst b/docs/spec/v2.rst index deb6d46ce..c1e12e121 100644 --- a/docs/spec/v2.rst +++ b/docs/spec/v2.rst @@ -3,5 +3,563 @@ Zarr Storage Specification Version 2 ==================================== -The V2 Specification has been migrated to its website → -https://zarr-specs.readthedocs.io/. +This document provides a technical specification of the protocol and format +used for storing Zarr arrays. The key words "MUST", "MUST NOT", "REQUIRED", +"SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and +"OPTIONAL" in this document are to be interpreted as described in `RFC 2119 +`_. + +Status +------ + +This specification is the latest version. See :ref:`spec` for previous +versions. + +.. _spec_v2_storage: + +Storage +------- + +A Zarr array can be stored in any storage system that provides a key/value +interface, where a key is an ASCII string and a value is an arbitrary sequence +of bytes, and the supported operations are read (get the sequence of bytes +associated with a given key), write (set the sequence of bytes associated with +a given key) and delete (remove a key/value pair). + +For example, a directory in a file system can provide this interface, where +keys are file names, values are file contents, and files can be read, written +or deleted via the operating system. Equally, an S3 bucket can provide this +interface, where keys are resource names, values are resource contents, and +resources can be read, written or deleted via HTTP. + +Below an "array store" refers to any system implementing this interface. + +.. _spec_v2_array: + +Arrays +------ + +.. _spec_v2_array_metadata: + +Metadata +~~~~~~~~ + +Each array requires essential configuration metadata to be stored, enabling +correct interpretation of the stored data. This metadata is encoded using JSON +and stored as the value of the ".zarray" key within an array store. + +The metadata resource is a JSON object. The following keys MUST be present +within the object: + +zarr_format + An integer defining the version of the storage specification to which the + array store adheres. +shape + A list of integers defining the length of each dimension of the array. +chunks + A list of integers defining the length of each dimension of a chunk of the + array. Note that all chunks within a Zarr array have the same shape. +dtype + A string or list defining a valid data type for the array. See also + the subsection below on data type encoding. +compressor + A JSON object identifying the primary compression codec and providing + configuration parameters, or ``null`` if no compressor is to be used. + The object MUST contain an ``"id"`` key identifying the codec to be used. +fill_value + A scalar value providing the default value to use for uninitialized + portions of the array, or ``null`` if no fill_value is to be used. +order + Either "C" or "F", defining the layout of bytes within each chunk of the + array. "C" means row-major order, i.e., the last dimension varies fastest; + "F" means column-major order, i.e., the first dimension varies fastest. +filters + A list of JSON objects providing codec configurations, or ``null`` if no + filters are to be applied. Each codec configuration object MUST contain a + ``"id"`` key identifying the codec to be used. + +The following keys MAY be present within the object: + +dimension_separator + If present, either the string ``"."`` or ``"/"`` defining the separator placed + between the dimensions of a chunk. If the value is not set, then the + default MUST be assumed to be ``"."``, leading to chunk keys of the form "0.0". + Arrays defined with ``"/"`` as the dimension separator can be considered to have + nested, or hierarchical, keys of the form "0/0" that SHOULD where possible + produce a directory-like structure. + +Other keys SHOULD NOT be present within the metadata object and SHOULD be +ignored by implementations. + +For example, the JSON object below defines a 2-dimensional array of 64-bit +little-endian floating point numbers with 10000 rows and 10000 columns, divided +into chunks of 1000 rows and 1000 columns (so there will be 100 chunks in total +arranged in a 10 by 10 grid). Within each chunk the data are laid out in C +contiguous order. Each chunk is encoded using a delta filter and compressed +using the Blosc compression library prior to storage:: + + { + "chunks": [ + 1000, + 1000 + ], + "compressor": { + "id": "blosc", + "cname": "lz4", + "clevel": 5, + "shuffle": 1 + }, + "dtype": "`. The format +consists of 3 parts: + +* One character describing the byteorder of the data (``"<"``: little-endian; + ``">"``: big-endian; ``"|"``: not-relevant) +* One character code giving the basic type of the array (``"b"``: Boolean (integer + type where all values are only True or False); ``"i"``: integer; ``"u"``: unsigned + integer; ``"f"``: floating point; ``"c"``: complex floating point; ``"m"``: timedelta; + ``"M"``: datetime; ``"S"``: string (fixed-length sequence of char); ``"U"``: unicode + (fixed-length sequence of Py_UNICODE); ``"V"``: other (void * – each item is a + fixed-size chunk of memory)) +* An integer specifying the number of bytes the type uses. + +The byte order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and +``"|S12"`` are valid data type encodings. + +For datetime64 ("M") and timedelta64 ("m") data types, these MUST also include the +units within square brackets. A list of valid units and their definitions are given in +the :ref:`NumPy documentation on Datetimes and Timedeltas +`. +For example, ``"`. Each +sub-list has the form ``[fieldname, datatype, shape]`` where ``shape`` +is optional. ``fieldname`` is a string, ``datatype`` is a string +specifying a simple data type (see above), and ``shape`` is a list of +integers specifying subarray shape. For example, the JSON list below +defines a data type composed of three single-byte unsigned integer +fields named "r", "g" and "b":: + + [["r", "|u1"], ["g", "|u1"], ["b", "|u1"]] + +For example, the JSON list below defines a data type composed of three +fields named "x", "y" and "z", where "x" and "y" each contain 32-bit +floats, and each item in "z" is a 2 by 2 array of floats:: + + [["x", "`_ +produces a sequence of bytes that begins with a 16-byte header followed by +compressed data. + +The compressed sequence of bytes for each chunk is stored under a key formed +from the index of the chunk within the grid of chunks representing the array. +To form a string key for a chunk, the indices are converted to strings and +concatenated with the period character (".") separating each index. For +example, given an array with shape (10000, 10000) and chunk shape (1000, 1000) +there will be 100 chunks laid out in a 10 by 10 grid. The chunk with indices +(0, 0) provides data for rows 0-999 and columns 0-999 and is stored under the +key "0.0"; the chunk with indices (2, 4) provides data for rows 2000-2999 and +columns 4000-4999 and is stored under the key "2.4"; etc. + +There is no need for all chunks to be present within an array store. If a chunk +is not present then it is considered to be in an uninitialized state. An +uninitialized chunk MUST be treated as if it was uniformly filled with the value +of the "fill_value" field in the array metadata. If the "fill_value" field is +``null`` then the contents of the chunk are undefined. + +Note that all chunks in an array have the same shape. If the length of any +array dimension is not exactly divisible by the length of the corresponding +chunk dimension then some chunks will overhang the edge of the array. The +contents of any chunk region falling outside the array are undefined. + +.. _spec_v2_array_filters: + +Filters +~~~~~~~ + +Optionally a sequence of one or more filters can be used to transform chunk +data prior to compression. When storing data, filters are applied in the order +specified in array metadata to encode data, then the encoded data are passed to +the primary compressor. When retrieving data, stored chunk data are +decompressed by the primary compressor then decoded using filters in the +reverse order. + +.. _spec_v2_hierarchy: + +Hierarchies +----------- + +.. _spec_v2_hierarchy_paths: + +Logical storage paths +~~~~~~~~~~~~~~~~~~~~~ + +Multiple arrays can be stored in the same array store by associating each array +with a different logical path. A logical path is simply an ASCII string. The +logical path is used to form a prefix for keys used by the array. For example, +if an array is stored at logical path "foo/bar" then the array metadata will be +stored under the key "foo/bar/.zarray", the user-defined attributes will be +stored under the key "foo/bar/.zattrs", and the chunks will be stored under +keys like "foo/bar/0.0", "foo/bar/0.1", etc. + +To ensure consistent behaviour across different storage systems, logical paths +MUST be normalized as follows: + +* Replace all backward slash characters ("\\\\") with forward slash characters + ("/") +* Strip any leading "/" characters +* Strip any trailing "/" characters +* Collapse any sequence of more than one "/" character into a single "/" + character + +The key prefix is then obtained by appending a single "/" character to the +normalized logical path. + +After normalization, if splitting a logical path by the "/" character results +in any path segment equal to the string "." or the string ".." then an error +MUST be raised. + +N.B., how the underlying array store processes requests to store values under +keys containing the "/" character is entirely up to the store implementation +and is not constrained by this specification. E.g., an array store could simply +treat all keys as opaque ASCII strings; equally, an array store could map +logical paths onto some kind of hierarchical storage (e.g., directories on a +file system). + +.. _spec_v2_hierarchy_groups: + +Groups +~~~~~~ + +Arrays can be organized into groups which can also contain other groups. A +group is created by storing group metadata under the ".zgroup" key under some +logical path. E.g., a group exists at the root of an array store if the +".zgroup" key exists in the store, and a group exists at logical path "foo/bar" +if the "foo/bar/.zgroup" key exists in the store. + +If the user requests a group to be created under some logical path, then groups +MUST also be created at all ancestor paths. E.g., if the user requests group +creation at path "foo/bar" then groups MUST be created at path "foo" and the +root of the store, if they don't already exist. + +If the user requests an array to be created under some logical path, then +groups MUST also be created at all ancestor paths. E.g., if the user requests +array creation at path "foo/bar/baz" then groups must be created at path +"foo/bar", path "foo", and the root of the store, if they don't already exist. + +The group metadata resource is a JSON object. The following keys MUST be present +within the object: + +zarr_format + An integer defining the version of the storage specification to which the + array store adheres. + +Other keys MUST NOT be present within the metadata object. + +The members of a group are arrays and groups stored under logical paths that +are direct children of the parent group's logical path. E.g., if groups exist +under the logical paths "foo" and "foo/bar" and an array exists at logical path +"foo/baz" then the members of the group at path "foo" are the group at path +"foo/bar" and the array at path "foo/baz". + +.. _spec_v2_attrs: + +Attributes +---------- + +An array or group can be associated with custom attributes, which are arbitrary +key/value pairs with application-specific meaning. Custom attributes are encoded +as a JSON object and stored under the ".zattrs" key within an array store. The +".zattrs" key does not have to be present, and if it is absent the attributes +should be treated as empty. + +For example, the JSON object below encodes three attributes named +"foo", "bar" and "baz":: + + { + "foo": 42, + "bar": "apples", + "baz": [1, 2, 3, 4] + } + +.. _spec_v2_examples: + +Examples +-------- + +Storing a single array +~~~~~~~~~~~~~~~~~~~~~~ + +Below is an example of storing a Zarr array, using a directory on the +local file system as storage. + +Create an array:: + + >>> import zarr + >>> store = zarr.DirectoryStore('data/example.zarr') + >>> a = zarr.create(shape=(20, 20), chunks=(10, 10), dtype='i4', + ... fill_value=42, compressor=zarr.Zlib(level=1), + ... store=store, overwrite=True) + +No chunks are initialized yet, so only the ".zarray" and ".zattrs" keys +have been set in the store:: + + >>> import os + >>> sorted(os.listdir('data/example.zarr')) + ['.zarray'] + +Inspect the array metadata:: + + >>> print(open('data/example.zarr/.zarray').read()) + { + "chunks": [ + 10, + 10 + ], + "compressor": { + "id": "zlib", + "level": 1 + }, + "dtype": ">> a[0:10, 0:10] = 1 + >>> sorted(os.listdir('data/example.zarr')) + ['.zarray', '0.0'] + +Set some more data:: + + >>> a[0:10, 10:20] = 2 + >>> a[10:20, :] = 3 + >>> sorted(os.listdir('data/example.zarr')) + ['.zarray', '0.0', '0.1', '1.0', '1.1'] + +Manually decompress a single chunk for illustration:: + + >>> import zlib + >>> buf = zlib.decompress(open('data/example.zarr/0.0', 'rb').read()) + >>> import numpy as np + >>> chunk = np.frombuffer(buf, dtype='>> chunk + array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32) + +Modify the array attributes:: + + >>> a.attrs['foo'] = 42 + >>> a.attrs['bar'] = 'apples' + >>> a.attrs['baz'] = [1, 2, 3, 4] + >>> sorted(os.listdir('data/example.zarr')) + ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1'] + >>> print(open('data/example.zarr/.zattrs').read()) + { + "bar": "apples", + "baz": [ + 1, + 2, + 3, + 4 + ], + "foo": 42 + } + +Storing multiple arrays in a hierarchy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Below is an example of storing multiple Zarr arrays organized into a group +hierarchy, using a directory on the local file system as storage. This storage +implementation maps logical paths onto directory paths on the file system, +however this is an implementation choice and is not required. + +Setup the store:: + + >>> import zarr + >>> store = zarr.DirectoryStore('data/group.zarr') + +Create the root group:: + + >>> root_grp = zarr.group(store, overwrite=True) + +The metadata resource for the root group has been created:: + + >>> import os + >>> sorted(os.listdir('data/group.zarr')) + ['.zgroup'] + +Inspect the group metadata:: + + >>> print(open('data/group.zarr/.zgroup').read()) + { + "zarr_format": 2 + } + +Create a sub-group:: + + >>> sub_grp = root_grp.create_group('foo') + +What has been stored:: + + >>> sorted(os.listdir('data/group.zarr')) + ['.zgroup', 'foo'] + >>> sorted(os.listdir('data/group.zarr/foo')) + ['.zgroup'] + +Create an array within the sub-group:: + + >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10)) + >>> a[:] = 42 + +Set a custom attributes:: + + >>> a.attrs['comment'] = 'answer to life, the universe and everything' + +What has been stored:: + + >>> sorted(os.listdir('data/group.zarr')) + ['.zgroup', 'foo'] + >>> sorted(os.listdir('data/group.zarr/foo')) + ['.zgroup', 'bar'] + >>> sorted(os.listdir('data/group.zarr/foo/bar')) + ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1'] + +Here is the same example using a Zip file as storage:: + + >>> store = zarr.ZipStore('data/group.zip', mode='w') + >>> root_grp = zarr.group(store) + >>> sub_grp = root_grp.create_group('foo') + >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10)) + >>> a[:] = 42 + >>> a.attrs['comment'] = 'answer to life, the universe and everything' + >>> store.close() + +What has been stored:: + + >>> import zipfile + >>> zf = zipfile.ZipFile('data/group.zip', mode='r') + >>> for name in sorted(zf.namelist()): + ... print(name) + .zgroup + foo/.zgroup + foo/bar/.zarray + foo/bar/.zattrs + foo/bar/0.0 + foo/bar/0.1 + foo/bar/1.0 + foo/bar/1.1 + +.. _spec_v2_changes: + +Changes +------- + +Version 2 clarifications +~~~~~~~~~~~~~~~~~~~~~~~~ + +The following changes have been made to the version 2 specification since it was +initially published to clarify ambiguities and add some missing information. + +* The specification now describes how bytes fill values should be encoded and + decoded for arrays with a fixed-length byte string data type (:issue:`165`, + :issue:`176`). + +* The specification now clarifies that units must be specified for datetime64 and + timedelta64 data types (:issue:`85`, :issue:`215`). + +* The specification now clarifies that the '.zattrs' key does not have to be present for + either arrays or groups, and if absent then custom attributes should be treated as + empty. + +* The specification now describes how structured datatypes with + subarray shapes and/or with nested structured data types are encoded + in array metadata (:issue:`111`, :issue:`296`). + +* Clarified the key/value pairs of custom attributes as "arbitrary" rather than + "simple". + +Changes from version 1 to version 2 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following changes were made between version 1 and version 2 of this specification: + +* Added support for storing multiple arrays in the same store and organising + arrays into hierarchies using groups. +* Array metadata is now stored under the ".zarray" key instead of the "meta" + key. +* Custom attributes are now stored under the ".zattrs" key instead of the + "attrs" key. +* Added support for filters. +* Changed encoding of "fill_value" field within array metadata. +* Changed encoding of compressor information within array metadata to be + consistent with representation of filter information. diff --git a/docs/spec/v3.rst b/docs/spec/v3.rst index 3d39f35ba..bd8852707 100644 --- a/docs/spec/v3.rst +++ b/docs/spec/v3.rst @@ -1,7 +1,7 @@ .. _spec_v3: Zarr Storage Specification Version 3 -==================================== +======================================================= The V3 Specification has been migrated to its website → https://zarr-specs.readthedocs.io/. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 619392a17..a40422490 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -774,7 +774,7 @@ the following code:: Any other compatible storage class could be used in place of :class:`zarr.storage.DirectoryStore` in the code examples above. For example, -here is an array stored directly into a ZIP archive, via the +here is an array stored directly into a Zip file, via the :class:`zarr.storage.ZipStore` class:: >>> store = zarr.ZipStore('data/example.zip', mode='w') @@ -798,12 +798,12 @@ Re-open and check that data have been written:: [42, 42, 42, ..., 42, 42, 42]], dtype=int32) >>> store.close() -Note that there are some limitations on how ZIP archives can be used, because items -within a ZIP archive cannot be updated in place. This means that data in the array +Note that there are some limitations on how Zip files can be used, because items +within a Zip file cannot be updated in place. This means that data in the array should only be written once and write operations should be aligned with chunk boundaries. Note also that the ``close()`` method must be called after writing any data to the store, otherwise essential records will not be written to the -underlying ZIP archive. +underlying zip file. Another storage alternative is the :class:`zarr.storage.DBMStore` class, added in Zarr version 2.2. This class allows any DBM-style database to be used for @@ -846,7 +846,7 @@ respectively require the `redis-py `_ and `pymongo `_ packages to be installed. For compatibility with the `N5 `_ data format, Zarr also provides -an N5 backend (this is currently an experimental feature). Similar to the ZIP storage class, an +an N5 backend (this is currently an experimental feature). Similar to the zip storage class, an :class:`zarr.n5.N5Store` can be instantiated directly:: >>> store = zarr.N5Store('data/example.n5') @@ -868,7 +868,7 @@ implementations of the ``MutableMapping`` interface for Amazon S3 (`S3Map Distributed File System (`HDFSMap `_) and Google Cloud Storage (`GCSMap -`_), which +`_), which can be used with Zarr. Here is an example using S3Map to read an array created previously:: @@ -1000,32 +1000,6 @@ separately from Zarr. .. _tutorial_copy: -Accessing ZIP archives on S3 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The built-in :class:`zarr.storage.ZipStore` will only work with paths on the local file-system; however -it is possible to access ZIP-archived Zarr data on the cloud via the `ZipFileSystem `_ -class from ``fsspec``. The following example demonstrates how to access -a ZIP-archived Zarr group on s3 using `s3fs `_ and ``ZipFileSystem``: - - >>> s3_path = "s3://path/to/my.zarr.zip" - >>> - >>> s3 = s3fs.S3FileSystem() - >>> f = s3.open(s3_path) - >>> fs = ZipFileSystem(f, mode="r") - >>> store = FSMap("", fs, check=False) - >>> - >>> # caching may improve performance when repeatedly reading the same data - >>> cache = zarr.storage.LRUStoreCache(store, max_size=2**28) - >>> z = zarr.group(store=cache) - -This store can also be generated with ``fsspec``'s handler chaining, like so: - - >>> store = zarr.storage.FSStore(url=f"zip::{s3_path}", mode="r") - -This can be especially useful if you have a very large ZIP-archived Zarr array or group on s3 -and only need to access a small portion of it. - Consolidating metadata ~~~~~~~~~~~~~~~~~~~~~~ @@ -1162,7 +1136,7 @@ re-compression, and so should be faster. E.g.:: └── spam (100,) int64 >>> new_root['foo/bar/baz'][:] array([ 0, 1, 2, ..., 97, 98, 99]) - >>> store2.close() # ZIP stores need to be closed + >>> store2.close() # zip stores need to be closed .. _tutorial_strings: diff --git a/pyproject.toml b/pyproject.toml index 2bc2b526e..b77260b00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,7 +109,6 @@ Homepage = "https://github.com/zarr-developers/zarr-python" exclude_lines = [ "pragma: no cover", "pragma: ${PY_MAJOR_VERSION} no cover", - '.*\.\.\.' # Ignore "..." lines ] [tool.coverage.run] diff --git a/v3-roadmap-and-design.md b/v3-roadmap-and-design.md deleted file mode 100644 index 696799e56..000000000 --- a/v3-roadmap-and-design.md +++ /dev/null @@ -1,429 +0,0 @@ -# Zarr Python Roadmap - -- Status: draft -- Author: Joe Hamman -- Created On: October 31, 2023 -- Input from: - - Davis Bennett / @d-v-b - - Norman Rzepka / @normanrz - - Deepak Cherian @dcherian - - Brian Davis / @monodeldiablo - - Oliver McCormack / @olimcc - - Ryan Abernathey / @rabernat - - Jack Kelly / @JackKelly - - Martin Durrant / @martindurant - -## Introduction - -This document lays out a design proposal for version 3.0 of the [Zarr-Python](https://zarr.readthedocs.io/en/stable/) package. A specific focus of the design is to bring Zarr-Python's API up to date with the [Zarr V3 specification](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html), with the hope of enabling the development of the many features and extensions that motivated the V3 Spec. The ideas presented here are expected to result in a major release of Zarr-Python (version 3.0) including significant a number of breaking API changes. -For clarity, “V3” will be used to describe the version of the Zarr specification and “3.0” will be used to describe the release tag of the Zarr-Python project. - -### Current status of V3 in Zarr-Python - -During the development of the V3 Specification, a [prototype implementation](https://github.com/zarr-developers/zarr-python/pull/898) was added to the Zarr-Python library. Since that implementation, the V3 spec evolved in significant ways and as a result, the Zarr-Python library is now out of sync with the approved spec. Downstream libraries (e.g. [Xarray](https://github.com/pydata/xarray)) have added support for this implementation and will need to migrate to the accepted spec when its available in Zarr-Python. - -## Goals - -- Provide a complete implementation of Zarr V3 through the Zarr-Python API -- Clear the way for exciting extensions / ZEPs (i.e. [sharding](https://zarr-specs.readthedocs.io/en/latest/v3/codecs/sharding-indexed/v1.0.html), [variable chunking](https://zarr.dev/zeps/draft/ZEP0003.html), etc.) -- Provide a developer API that can be used to implement and register V3 extensions -- Improve the performance of Zarr-Python by streamlining the interface between the Store layer and higher level APIs (e.g. Groups and Arrays) -- Clean up the internal and user facing APIs -- Improve code quality and robustness (e.g. achieve 100% type hint coverage) -- Align the Zarr-Python array API with the [array API Standard](https://data-apis.org/array-api/latest/) - -## Examples of what 3.0 will enable? -1. Reading and writing V3 spec-compliant groups and arrays -2. V3 extensions including sharding and variable chunking. -3. Improved performance by leveraging concurrency when creating/reading/writing to stores (imagine a `create_hierarchy(zarr_objects)` function). -4. User-developed extensions (e.g. storage-transformers) can be registered with Zarr-Python at runtime - -## Non-goals (of this document) - -- Implementation of any unaccepted Zarr V3 extensions -- Major revisions to the Zarr V3 spec - -## Requirements - -1. Read and write spec compliant V2 and V3 data -2. Limit unnecessary traffic to/from the store -3. Cleanly define the Array/Group/Store abstractions -4. Cleanly define how V2 will be supported going forward -5. Provide a clear roadmap to help users upgrade to 3.0 -6. Developer tools / hooks for registering extensions - -## Design - -### Async API - -Zarr-Python is an IO library. As such, supporting concurrent action against the storage layer is critical to achieving acceptable performance. The Zarr-Python 2 was not designed with asynchronous computation in mind and as a result has struggled to effectively leverage the benefits of concurrency. At one point, `getitems` and `setitems` support was added to the Zarr store model but that is only used for operating on a set of chunks in a single variable. - -With Zarr-Python 3.0, we have the opportunity to revisit this design. The proposal here is as follows: - -1. The `Store` interface will be entirely async. -2. On top of the async `Store` interface, we will provide an `AsyncArray` and `AsyncGroup` interface. -3. Finally, the primary user facing API will be synchronous `Array` and `Group` classes that wrap the async equivalents. - -**Examples** - -- **Store** - - ```python - class Store: - ... - async def get(self, key: str) -> bytes: - ... - async def get_partial_values(self, key_ranges: List[Tuple[str, Tuple[int, Optional[int]]]]) -> bytes: - ... - # (no sync interface here) - ``` -- **Array** - - ```python - class AsyncArray: - ... - - async def getitem(self, selection: Selection) -> np.ndarray: - # the core logic for getitem goes here - - class Array: - _async_array: AsyncArray - - def __getitem__(self, selection: Selection) -> np.ndarray: - return sync(self._async_array.getitem(selection)) - ``` -- **Group** - - ```python - class AsyncGroup: - ... - - async def create_group(self, path: str, **kwargs) -> AsyncGroup: - # the core logic for create_group goes here - - class Group: - _async_group: AsyncGroup - - def create_group(self, path: str, **kwargs) -> Group: - return sync(self._async_group.create_group(path, **kwargs)) - ``` -**Internal Synchronization API** - -With the `Store` and core `AsyncArray`/ `AsyncGroup` classes being predominantly async, Zarr-Python will need an internal API to provide a synchronous API. The proposal here is to use the approach in [fsspec](https://github.com/fsspec/filesystem_spec/blob/master/fsspec/asyn.py) to provide a high-level `sync` function that takes an `awaitable` and runs it in its managed IO Loop / thread. - -**FAQ** -1. Why two levels of Arrays/groups? - a. First, this is an intentional decision and departure from the current Zarrita implementation - b. The idea is that users rarely want to mix interfaces. Either they are working within an async context (currently quite rare) or they are in a typical synchronous context. - c. Splitting the two will allow us to clearly define behavior on the `AsyncObj` and simply wrap it in the `SyncObj`. -2. What if a store is only has a synchronous backend? - a. First off, this is expected to be a fairly rare occurrence. Most storage backends have async interfaces. - b. But in the event a storage backend doesn’t have a async interface, there is nothing wrong with putting synchronous code in `async` methods. There are approaches to enabling concurrent action through wrappers like AsyncIO's `loop.run_in_executor` ([ref 1](https://stackoverflow.com/questions/38865050/is-await-in-python3-cooperative-multitasking ), [ref 2](https://stackoverflow.com/a/43263397/732596), [ref 3](https://bbc.github.io/cloudfit-public-docs/asyncio/asyncio-part-5.html), [ref 4](https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.run_in_executor). -3. Will Zarr help manage the async contexts encouraged by some libraries (e.g. [AioBotoCore](https://aiobotocore.readthedocs.io/en/latest/tutorial.html#using-botocore))? - a. Many async IO libraries require entering an async context before interacting with the API. We expect some experimentation to be needed here but the initial design will follow something close to what fsspec does ([example in s3fs](https://github.com/fsspec/s3fs/blob/949442693ec940b35cda3420c17a864fbe426567/s3fs/core.py#L527)). -4. Why not provide a synchronous Store interface? - a. We could but this design is simpler. It would mean supporting it in the `AsyncGroup` and `AsyncArray` classes which, may be more trouble than its worth. Storage backends that do not have an async API will be encouraged to wrap blocking calls in an async wrapper (e.g. `loop.run_in_executor`). - -### Store API - -The `Store` API is specified directly in the V3 specification. All V3 stores should implement this abstract API, omitting Write and List support as needed. As described above, all stores will be expected to expose the required methods as async methods. - -**Example** - -```python -class ReadWriteStore: - ... - async def get(self, key: str) -> bytes: - ... - - async def get_partial_values(self, key_ranges: List[Tuple[str, int, int]) -> bytes: - ... - - async def set(self, key: str, value: Union[bytes, bytearray, memoryview]) -> None: - ... # required for writable stores - - async def set_partial_values(self, key_start_values: List[Tuple[str, int, Union[bytes, bytearray, memoryview]]]) -> None: - ... # required for writable stores - - async def list(self) -> List[str]: - ... # required for listable stores - - async def list_prefix(self, prefix: str) -> List[str]: - ... # required for listable stores - - async def list_dir(self, prefix: str) -> List[str]: - ... # required for listable stores - - # additional (optional methods) - async def getsize(self, prefix: str) -> int: - ... - - async def rename(self, src: str, dest: str) -> None - ... - -``` - -Recognizing that there are many Zarr applications today that rely on the `MutableMapping` interface supported by Zarr-Python 2, a wrapper store will be developed to allow existing stores to plug directly into this API. - -### Array API - -The user facing array interface will implement a subset of the [Array API Standard](https://data-apis.org/array-api/latest/). Most of the computational parts of the Array API Standard don’t fit into Zarr right now. That’s okay. What matters most is that we ensure we can give downstream applications a compliant API. - -*Note, Zarr already does most of this so this is more about formalizing the relationship than a substantial change in API.* - -| | Included | Not Included | Unknown / Maybe possible? | -| --- | --- | --- | --- | -| Attributes | `dtype` | `mT` | `device` | -| | `ndim` | `T` | | -| | `shape` | | | -| | `size` | | | -| Methods | `__getitem__` | `__array_namespace__` | `to_device` | -| | `__setitem__` | `__abs__` | `__bool__` | -| | `__eq__` | `__add__` | `__complex__` | -| | `__bool__` | `__and__` | `__dlpack__` | -| | | `__floordiv__` | `__dlpack_device__` | -| | | `__ge__` | `__float__` | -| | | `__gt__` | `__index__` | -| | | `__invert__` | `__int__` | -| | | `__le__` | | -| | | `__lshift__` | | -| | | `__lt__` | | -| | | `__matmul__` | | -| | | `__mod__` | | -| | | `__mul__` | | -| | | `__ne__` | | -| | | `__neg__` | | -| | | `__or__` | | -| | | `__pos__` | | -| | | `__pow__` | | -| | | `__rshift__` | | -| | | `__sub__` | | -| | | `__truediv__` | | -| | | `__xor__` | | -| Creation functions (`zarr.creation`) | `zeros` | | `arange` | -| | `zeros_like` | | `asarray` | -| | `ones` | | `eye` | -| | `ones_like` | | `from_dlpack` | -| | `full` | | `linspace` | -| | `full_like` | | `meshgrid` | -| | `empty` | | `tril` | -| | `empty_like` | | `triu` | - -In addition to the core array API defined above, the Array class should have the following Zarr specific properties: - -- `.metadata` (see Metadata Interface below) -- `.attrs` - (pull from metadata object) -- `.info` - (pull from existing property †) - -*† In Zarr-Python 2, the info property lists the store to identify initialized chunks. By default this will be turned off in 3.0 but will be configurable.* - -**Indexing** - -Zarr-Python currently supports `__getitem__` style indexing and the special `oindex` and `vindex` indexers. These are not part of the current Array API standard (see [data-apis/array-api\#669](https://github.com/data-apis/array-api/issues/669)) but they have been [proposed as a NEP](https://numpy.org/neps/nep-0021-advanced-indexing.html). Zarr-Python will maintain these in 3.0. - -We are also exploring a new high-level indexing API that will enabled optimized batch/concurrent loading of many chunks. We expect this to be important to enable performant loading of data in the context of sharding. See [this discussion](https://github.com/zarr-developers/zarr-python/discussions/1569) for more detail. - -Concurrent indexing across multiple arrays will be possible using the AsyncArray API. - -**Async and Sync Array APIs** - -Most the logic to support Zarr Arrays will live in the `AsyncArray` class. There are a few notable differences that should be called out. - -| Sync Method | Async Method | -| --- | --- | -| `__getitem__` | `getitem` | -| `__setitem__` | `setitem` | -| `__eq__` | `equals` | - -**Metadata interface** - -Zarr-Python 2.* closely mirrors the V2 spec metadata schema in the Array and Group classes. In 3.0, we plan to move the underlying metadata representation to a separate interface (e.g. `Array.metadata`). This interface will return either a `V2ArrayMetadata` or `V3ArrayMetadata` object (both will inherit from a parent `ArrayMetadataABC` class. The `V2ArrayMetadata` and `V3ArrayMetadata` classes will be responsible for producing valid JSON representations of their metadata, and yielding a consistent view to the `Array` or `Group` class. - -### Group API - -The main question is how closely we should follow the existing Zarr-Python implementation / `MutableMapping` interface. The table below shows the primary `Group` methods in Zarr-Python 2 and attempts to identify if and how they would be implemented in 3.0. - -| V2 Group Methods | `AsyncGroup` | `Group` | `h5py_compat.Group`` | -| --- | --- | --- | --- | -| `__len__` | `length` | `__len__` | `__len__` | -| `__iter__` | `__aiter__` | `__iter__` | `__iter__` | -| `__contains__` | `contains` | `__contains__` | `__contains__` | -| `__getitem__` | `getitem` | `__getitem__` | `__getitem__` | -| `__enter__` | N/A | N/A | `__enter__` | -| `__exit__` | N/A | N/A | `__exit__` | -| `group_keys` | `group_keys` | `group_keys` | N/A | -| `groups` | `groups` | `groups` | N/A | -| `array_keys` | `array_key` | `array_keys` | N/A | -| `arrays` | `arrays`* | `arrays` | N/A | -| `visit` | ? | ? | `visit` | -| `visitkeys` | ? | ? | ? | -| `visitvalues` | ? | ? | ? | -| `visititems` | ? | ? | `visititems` | -| `tree` | `tree` | `tree` | `Both` | -| `create_group` | `create_group` | `create_group` | `create_group` | -| `require_group` | N/A | N/A | `require_group` | -| `create_groups` | ? | ? | N/A | -| `require_groups` | ? | ? | ? | -| `create_dataset` | N/A | N/A | `create_dataset` | -| `require_dataset` | N/A | N/A | `require_dataset` | -| `create` | `create_array` | `create_array` | N/A | -| `empty` | `empty` | `empty` | N/A | -| `zeros` | `zeros` | `zeros` | N/A | -| `ones` | `ones` | `ones` | N/A | -| `full` | `full` | `full` | N/A | -| `array` | `create_array` | `create_array` | N/A | -| `empty_like` | `empty_like` | `empty_like` | N/A | -| `zeros_like` | `zeros_like` | `zeros_like` | N/A | -| `ones_like` | `ones_like` | `ones_like` | N/A | -| `full_like` | `full_like` | `full_like` | N/A | -| `move` | `move` | `move` | `move` | - -**`zarr.h5compat.Group`** - -Zarr-Python 2.* made an attempt to align its API with that of [h5py](https://docs.h5py.org/en/stable/index.html). With 3.0, we will relax this alignment in favor of providing an explicit compatibility module (`zarr.h5py_compat`). This module will expose the `Group` and `Dataset` APIs that map to Zarr-Python’s `Group` and `Array` objects. - -### Creation API - -Zarr-Python 2.* bundles together the creation and serialization of Zarr objects. Zarr-Python 3.* will make it possible to create objects in memory separate from serializing them. This will specifically enable writing hierarchies of Zarr objects in a single batch step. For example: - -```python - -arr1 = Array(shape=(10, 10), path="foo/bar", dtype="i4", store=store) -arr2 = Array(shape=(10, 10), path="foo/spam", dtype="f8", store=store) - -arr1.save() -arr2.save() - -# or equivalently - -zarr.save_many([arr1 ,arr2]) -``` - -*Note: this batch creation API likely needs additional design effort prior to implementation.* - -### Plugin API - -Zarr V3 was designed to be extensible at multiple layers. Zarr-Python will support these extensions through a combination of [Abstract Base Classes](https://docs.python.org/3/library/abc.html) (ABCs) and [Entrypoints](https://packaging.python.org/en/latest/specifications/entry-points/). - -**ABCs** - -Zarr V3 will expose Abstract base classes for the following objects: - -- `Store`, `ReadStore`, `ReadWriteStore`, `ReadListStore`, and `ReadWriteListStore` -- `BaseArray`, `SynchronousArray`, and `AsynchronousArray` -- `BaseGroup`, `SynchronousGroup`, and `AsynchronousGroup` -- `Codec`, `ArrayArrayCodec`, `ArrayBytesCodec`, `BytesBytesCodec` - -**Entrypoints** - -Lots more thinking here but the idea here is to provide entrypoints for `data type`, `chunk grid`, `chunk key encoding`, `codecs`, `storage_transformers` and `stores`. These might look something like: - -``` -entry_points=""" - [zarr.codecs] - blosc_codec=codec_plugin:make_blosc_codec - zlib_codec=codec_plugin:make_zlib_codec -""" -``` - -### Python type hints and static analysis - -Target 100% Mypy coverage in 3.0 source. - -### Observability - -A persistent problem in Zarr-Python is diagnosing problems that span many parts of the stack. To address this in 3.0, we will add a basic logging framework that can be used to debug behavior at various levels of the stack. We propose to add the separate loggers for the following namespaces: - -- `array` -- `group` -- `store` -- `codec` - -These should be documented such that users know how to activate them and developers know how to use them when developing extensions. - -### Dependencies - -Today, Zarr-Python has the following required dependencies: - -```python -dependencies = [ - 'asciitree', - 'numpy>=1.20,!=1.21.0', - 'fasteners', - 'numcodecs>=0.10.0', -] -``` - -What other dependencies should be considered? - -1. Attrs - Zarrita makes extensive use of the Attrs library -2. Fsspec - Zarrita has a hard dependency on Fsspec. This could be easily relaxed though. - -## Breaking changes relative to Zarr-Python 2.* - -1. H5py compat moved to a stand alone module? -2. `Group.__getitem__` support moved to `Group.members.__getitem__`? -3. Others? - -## Open questions - -1. How to treat V2 - a. Note: Zarrita currently implements a separate `V2Array` and `V3Array` classes. This feels less than ideal. - b. We could easily convert metadata from v2 to the V3 Array, but what about writing? - c. Ideally, we don’t have completely separate code paths. But if its too complicated to support both within one interface, its probably better. -2. How and when to remove the current implementation of V3. - a. It's hidden behind a hard-to-use feature flag so we probably don't need to do anything. -4. How to model runtime configuration? -5. Which extensions belong in Zarr-Python and which belong in separate packages? - a. We don't need to take a strong position on this here. It's likely that someone will want to put Sharding in. That will be useful to develop in parallel because it will give us a good test case for the plugin interface. - -## Testing - -Zarr-python 3.0 adds a major new dimension to Zarr: Async support. This also comes with a compatibility risk, we will need to thoroughly test support in key execution environments. Testing plan: -- Reuse the existing test suite for testing the `v3` API. - - `xfail` tests that expose breaking changes with `3.0 - breaking change` description. This will help identify additional and/or unintentional breaking changes - - Rework tests that were only testing internal APIs. -- Add a set of functional / integration tests targeting real-world workflows in various contexts (e.g. w/ Dask) - -## Development process - -Zarr-Python 3.0 will introduce a number of new APIs and breaking changes to existing APIs. In order to facilitate ongoing support for Zarr-Python 2.*, we will take on the following development process: - -- Create a `v3` branch that can be use for developing the core functionality apart from the `main` branch. This will allow us to support ongoing work and bug fixes on the `main` branch. -- Put the `3.0` APIs inside a `zarr.v3` module. Imports from this namespace will all be new APIs that users can develop and test against once the `v3` branch is merged to `main`. -- Kickstart the process by pulling in the current state of `zarrita` - which has many of the features described in this design. -- Release a series of 2.* releases with the `v3` namespace -- When `v3` is complete, move contents of `v3` to the package root - -**Milestones** - -Below are a set of specific milestones leading toward the completion of this process. As work begins, we expect this list to grow in specificity. - -1. Port current version of Zarrita to Zarr-Python -2. Formalize Async interface by splitting `Array` and `Group` objects into Sync and Async versions -4. Implement "fancy" indexing operations on the `AsyncArray` -6. Implement an abstract base class for the `Store` interface and a wrapper `Store` to make use of existing `MutableMapping` stores. -7. Rework the existing unit test suite to use the `v3` namespace. -8. Develop a plugin interface for extensions -9. Develop a set of functional and integration tests -10. Work with downstream libraries (Xarray, Dask, etc.) to test new APIs - -## TODOs - -The following subjects are not covered in detail above but perhaps should be. Including them here so they are not forgotten. - -1. [Store] Should Zarr provide an API for caching objects after first read/list/etc. Read only stores? -2. [Array] buffer protocol support -3. [Array] `meta_array` support -4. [Extensions] Define how Zarr-Python will consume the various plugin types -5. [Misc] H5py compatibility requires a bit more work and a champion to drive it forward. -6. [Misc] Define `chunk_store` API in 3.0 -7. [Misc] Define `synchronizer` API in 3.0 - -## References - -1. [Zarr-Python repository](https://github.com/zarr-developers/zarr-python) -2. [Zarr core specification (version 3.0) — Zarr specs documentation](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#) -3. [Zarrita repository](https://github.com/scalableminds/zarrita) -4. [Async-Zarr](https://github.com/martindurant/async-zarr) -5. [Zarr-Python Discussion Topic](https://github.com/zarr-developers/zarr-python/discussions/1569)