From 4d49df6cf14c9dab1ed650dbbe0d7c2597fc4835 Mon Sep 17 00:00:00 2001 From: Matthew Iannucci Date: Mon, 6 Jan 2025 09:15:57 -0500 Subject: [PATCH 01/16] Start sync with zarr 3 rc1 --- Cargo.lock | 4 +-- docs/docs/icechunk-python/dask.md | 2 +- icechunk-python/Cargo.toml | 4 +-- icechunk-python/pyproject.toml | 4 +-- icechunk-python/tests/test_can_read_old.py | 6 ++-- icechunk-python/tests/test_concurrency.py | 2 +- icechunk-python/tests/test_config.py | 6 ++-- icechunk-python/tests/test_conflicts.py | 2 +- .../tests/test_distributed_writers.py | 2 +- icechunk-python/tests/test_regressions.py | 6 ++-- icechunk-python/tests/test_timetravel.py | 2 +- icechunk-python/tests/test_virtual_ref.py | 4 +-- icechunk-python/tests/test_zarr/test_array.py | 30 +++++++++---------- icechunk/Cargo.toml | 2 +- 14 files changed, 38 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d9c6c41f..e39fe078 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1314,7 +1314,7 @@ dependencies = [ [[package]] name = "icechunk" -version = "0.1.0-alpha.8" +version = "0.1.0-alpha.9" dependencies = [ "async-recursion", "async-stream", @@ -1352,7 +1352,7 @@ dependencies = [ [[package]] name = "icechunk-python" -version = "0.1.0-alpha.8" +version = "0.1.0-alpha.9" dependencies = [ "async-stream", "async-trait", diff --git a/docs/docs/icechunk-python/dask.md b/docs/docs/icechunk-python/dask.md index c83de460..ce3df9fa 100644 --- a/docs/docs/icechunk-python/dask.md +++ b/docs/docs/icechunk-python/dask.md @@ -47,7 +47,7 @@ group = zarr.group(store=icechunk_sesion.store(), overwrite=True) zarray = group.create_array( "array", shape=shape, - chunk_shape=zarr_chunks, + chunks=zarr_chunks, dtype="f8", fill_value=float("nan"), ) diff --git a/icechunk-python/Cargo.toml b/icechunk-python/Cargo.toml index a71a548b..08948dc7 100644 --- a/icechunk-python/Cargo.toml +++ b/icechunk-python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "icechunk-python" -version = "0.1.0-alpha.8" +version = "0.1.0-alpha.9" description = "Transactional storage engine for Zarr designed for use on cloud object storage" readme = "../README.md" repository = "https://github.com/earth-mover/icechunk" @@ -21,7 +21,7 @@ crate-type = ["cdylib"] bytes = "1.8.0" chrono = { version = "0.4.38" } futures = "0.3.31" -icechunk = { path = "../icechunk", version = "0.1.0-alpha.8" } +icechunk = { path = "../icechunk", version = "0.1.0-alpha.9" } pyo3 = { version = "0.23", features = [ "chrono", # do we really need this one? diff --git a/icechunk-python/pyproject.toml b/icechunk-python/pyproject.toml index 5a5ba0db..a38b5f24 100644 --- a/icechunk-python/pyproject.toml +++ b/icechunk-python/pyproject.toml @@ -16,11 +16,11 @@ classifiers = [ license = { text = "Apache-2.0" } dynamic = ["version"] -dependencies = ["zarr==3.0.0b3"] +dependencies = ["zarr==3.0.0rc1"] [tool.poetry] name = "icechunk" -version = "0.1.0-alpha.8" +version = "0.1.0-alpha.9" description = "Icechunk Python" authors = ["Earthmover "] readme = "../README.md" diff --git a/icechunk-python/tests/test_can_read_old.py b/icechunk-python/tests/test_can_read_old.py index 77c1bd32..edb171cb 100644 --- a/icechunk-python/tests/test_can_read_old.py +++ b/icechunk-python/tests/test_can_read_old.py @@ -74,7 +74,7 @@ async def write_a_test_repo() -> None: big_chunks = group1.create_array( "big_chunks", shape=(10, 10), - chunk_shape=(5, 5), + chunks=(5, 5), dtype="float32", fill_value=float("nan"), attributes={"this": "is a nice array", "icechunk": 1, "size": 42.0}, @@ -84,7 +84,7 @@ async def write_a_test_repo() -> None: small_chunks = group1.create_array( "small_chunks", shape=(5), - chunk_shape=(1), + chunks=(1), dtype="int8", fill_value=8, attributes={"this": "is a nice array", "icechunk": 1, "size": 42.0}, @@ -144,7 +144,7 @@ async def write_a_test_repo() -> None: group5.create_array( "inner", shape=(10, 10), - chunk_shape=(5, 5), + chunks=(5, 5), dtype="float32", fill_value=float("nan"), attributes={"this": "is a nice array", "icechunk": 1, "size": 42.0}, diff --git a/icechunk-python/tests/test_concurrency.py b/icechunk-python/tests/test_concurrency.py index db4ab7c4..06d0f185 100644 --- a/icechunk-python/tests/test_concurrency.py +++ b/icechunk-python/tests/test_concurrency.py @@ -50,7 +50,7 @@ async def test_concurrency() -> None: group = zarr.group(store=store, overwrite=True) array = group.create_array( - "array", shape=(N, N), chunk_shape=(1, 1), dtype="f8", fill_value=1e23 + "array", shape=(N, N), chunks=(1, 1), dtype="f8", fill_value=1e23 ) barrier = asyncio.Barrier(2 * N * N + 1) diff --git a/icechunk-python/tests/test_config.py b/icechunk-python/tests/test_config.py index 2bdd87e8..aa6fcf4a 100644 --- a/icechunk-python/tests/test_config.py +++ b/icechunk-python/tests/test_config.py @@ -33,7 +33,7 @@ def test_no_inline_chunks(tmp_store: tuple[icechunk.IcechunkStore, str]) -> None shape=(10), dtype="int64", zarr_format=3, - chunk_shape=(1), + chunks=(1), fill_value=-1, ) array[:] = 42 @@ -55,7 +55,7 @@ def test_inline_chunks(tmp_store: tuple[icechunk.IcechunkStore, str]) -> None: shape=(10), dtype="int32", zarr_format=3, - chunk_shape=(1), + chunks=(1), fill_value=-1, ) @@ -72,7 +72,7 @@ def test_inline_chunks(tmp_store: tuple[icechunk.IcechunkStore, str]) -> None: shape=(10), dtype="int64", zarr_format=3, - chunk_shape=(1), + chunks=(1), fill_value=-1, ) diff --git a/icechunk-python/tests/test_conflicts.py b/icechunk-python/tests/test_conflicts.py index 3f31c3f9..6f62b30d 100644 --- a/icechunk-python/tests/test_conflicts.py +++ b/icechunk-python/tests/test_conflicts.py @@ -19,7 +19,7 @@ def repo(tmpdir: Path) -> icechunk.Repository: root = zarr.group(store=store) root.create_group("foo/bar") root.create_array( - "foo/bar/some-array", shape=(10, 10), chunk_shape=(1, 1), dtype="i4" + "foo/bar/some-array", shape=(10, 10), chunks=(1, 1), dtype="i4" ) session.commit("commit 1") diff --git a/icechunk-python/tests/test_distributed_writers.py b/icechunk-python/tests/test_distributed_writers.py index 26f49711..176a7bc7 100644 --- a/icechunk-python/tests/test_distributed_writers.py +++ b/icechunk-python/tests/test_distributed_writers.py @@ -62,7 +62,7 @@ async def test_distributed_writers() -> None: zarray = group.create_array( "array", shape=shape, - chunk_shape=(CHUNK_DIM_SIZE, CHUNK_DIM_SIZE), + chunks=(CHUNK_DIM_SIZE, CHUNK_DIM_SIZE), dtype="f8", fill_value=float("nan"), ) diff --git a/icechunk-python/tests/test_regressions.py b/icechunk-python/tests/test_regressions.py index e843513f..0d5927b4 100644 --- a/icechunk-python/tests/test_regressions.py +++ b/icechunk-python/tests/test_regressions.py @@ -55,8 +55,8 @@ async def test_issue_418() -> None: store = session.store root = zarr.Group.from_store(store=store, zarr_format=3) - time = root.require_array(name="time", shape=((2,)), chunk_shape=((1,)), dtype="i4") - root.require_array(name="lon", shape=((1,)), chunk_shape=((1,)), dtype="i4") + time = root.require_array(name="time", shape=((2,)), chunks=((1,)), dtype="i4") + root.require_array(name="lon", shape=((1,)), chunks=((1,)), dtype="i4") # Set longitude store.set_virtual_ref( @@ -81,7 +81,7 @@ async def test_issue_418() -> None: root = zarr.Group.open(store=store) time = cast(zarr.core.array.Array, root["time"]) - root.require_array(name="lon", shape=((1,)), chunk_shape=((1,)), dtype="i4") + root.require_array(name="lon", shape=((1,)), chunks=((1,)), dtype="i4") # resize the array and append a new chunk time.resize((3,)) diff --git a/icechunk-python/tests/test_timetravel.py b/icechunk-python/tests/test_timetravel.py index 74748db0..b5720f18 100644 --- a/icechunk-python/tests/test_timetravel.py +++ b/icechunk-python/tests/test_timetravel.py @@ -19,7 +19,7 @@ def test_timetravel() -> None: group = zarr.group(store=store, overwrite=True) air_temp = group.create_array( - "air_temp", shape=(1000, 1000), chunk_shape=(100, 100), dtype="i4" + "air_temp", shape=(1000, 1000), chunks=(100, 100), dtype="i4" ) air_temp[:, :] = 42 diff --git a/icechunk-python/tests/test_virtual_ref.py b/icechunk-python/tests/test_virtual_ref.py index e1a50eb5..fc097c46 100644 --- a/icechunk-python/tests/test_virtual_ref.py +++ b/icechunk-python/tests/test_virtual_ref.py @@ -56,7 +56,7 @@ async def test_write_minio_virtual_refs() -> None: session = repo.writable_session("main") store = session.store - array = zarr.Array.create(store, shape=(5, 1, 3), chunk_shape=(1, 1, 1), dtype="i4") + array = zarr.Array.create(store, shape=(5, 1, 3), chunks=(1, 1, 1), dtype="i4") # We add the virtual chunk refs without checksum, with the right etag, and with the wrong wrong etag and datetime. # This way we can check retrieval operations that should fail @@ -198,7 +198,7 @@ async def test_from_s3_public_virtual_refs(tmpdir: Path) -> None: root = zarr.Group.from_store(store=store, zarr_format=3) year = root.require_array( - name="year", shape=((72,)), chunk_shape=((72,)), dtype="float32" + name="year", shape=((72,)), chunks=((72,)), dtype="float32" ) store.set_virtual_ref( diff --git a/icechunk-python/tests/test_zarr/test_array.py b/icechunk-python/tests/test_zarr/test_array.py index 9e180fe7..c04b2da6 100644 --- a/icechunk-python/tests/test_zarr/test_array.py +++ b/icechunk-python/tests/test_zarr/test_array.py @@ -10,7 +10,7 @@ from icechunk import IcechunkStore from tests.conftest import parse_repo -from zarr import Array, Group +from zarr import Array, Group, create_array from zarr.core.buffer import default_buffer_prototype from zarr.core.common import ZarrFormat from zarr.errors import ContainsArrayError, ContainsGroupError @@ -27,12 +27,12 @@ def store(request: pytest.FixtureRequest, tmpdir: Path) -> IcechunkStore: @pytest.mark.parametrize("store", ["memory"], indirect=["store"]) @pytest.mark.parametrize("zarr_format", [3]) -@pytest.mark.parametrize("exists_ok", [True, False]) +@pytest.mark.parametrize("overwrite", [True, False]) @pytest.mark.parametrize("extant_node", ["array", "group"]) def test_array_creation_existing_node( store: IcechunkStore, zarr_format: ZarrFormat, - exists_ok: bool, + overwrite: bool, extant_node: Literal["array", "group"], ) -> None: """ @@ -53,25 +53,25 @@ def test_array_creation_existing_node( new_shape = (2, 2) new_dtype = "float32" - if exists_ok: + if overwrite: # This is currently not supported by IcechunkStore - pytest.xfail("IcechunkStore does not support exists_ok=True") - # arr_new = Array.create( + pytest.xfail("IcechunkStore does not support overwrite=True") + # arr_new = create_array( # spath / "extant", # shape=new_shape, # dtype=new_dtype, - # exists_ok=exists_ok, + # overwrite=overwrite, # zarr_format=zarr_format, # ) # assert arr_new.shape == new_shape # assert arr_new.dtype == new_dtype else: with pytest.raises(expected_exception): - Array.create( + create_array( spath / "extant", shape=new_shape, dtype=new_dtype, - exists_ok=exists_ok, + overwrite=overwrite, zarr_format=zarr_format, ) @@ -80,7 +80,7 @@ def test_array_creation_existing_node( @pytest.mark.parametrize("store", ["local"], indirect=["store"]) @pytest.mark.parametrize("zarr_format", [3]) def test_serializable_sync_array(store: IcechunkStore, zarr_format: ZarrFormat) -> None: - expected = Array.create( + expected = create_array( store=store, shape=(100,), chunks=(10,), zarr_format=zarr_format, dtype="i4" ) expected[:] = list(range(100)) @@ -104,12 +104,12 @@ def test_array_v3_fill_value( store: IcechunkStore, fill_value: int, dtype_str: str ) -> None: shape = (10,) - arr = Array.create( + arr = create_array( store=store, shape=shape, dtype=dtype_str, zarr_format=3, - chunk_shape=shape, + chunks=shape, fill_value=fill_value, ) @@ -120,12 +120,12 @@ def test_array_v3_fill_value( @pytest.mark.parametrize("store", ["memory"], indirect=True) async def test_array_v3_nan_fill_value(store: IcechunkStore) -> None: shape = (10,) - arr = Array.create( + arr = create_array( store=store, shape=shape, dtype=np.float64, zarr_format=3, - chunk_shape=shape, + chunks=shape, fill_value=np.nan, ) arr[:] = np.nan @@ -151,7 +151,7 @@ async def test_array_v3_nan_fill_value(store: IcechunkStore) -> None: async def test_special_complex_fill_values_roundtrip( store: IcechunkStore, fill_value: Any, expected: list[Any] ) -> None: - Array.create(store=store, shape=(1,), dtype=np.complex64, fill_value=fill_value) + create_array(store=store, shape=(1,), dtype=np.complex64, fill_value=fill_value) content = await store.get("zarr.json", prototype=default_buffer_prototype()) assert content is not None actual = json.loads(content.to_bytes()) diff --git a/icechunk/Cargo.toml b/icechunk/Cargo.toml index 3719d7f8..f0d559ae 100644 --- a/icechunk/Cargo.toml +++ b/icechunk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "icechunk" -version = "0.1.0-alpha.8" +version = "0.1.0-alpha.9" description = "Transactional storage engine for Zarr designed for use on cloud object storage" readme = "../README.md" repository = "https://github.com/earth-mover/icechunk" From ddef0b68787c6b8156a27527f694c6531cf23b9a Mon Sep 17 00:00:00 2001 From: Matthew Iannucci Date: Mon, 6 Jan 2025 09:22:52 -0500 Subject: [PATCH 02/16] More tests updated --- icechunk-python/tests/test_zarr/test_group.py | 87 ++++++++++--------- 1 file changed, 46 insertions(+), 41 deletions(-) diff --git a/icechunk-python/tests/test_zarr/test_group.py b/icechunk-python/tests/test_zarr/test_group.py index 793c1e36..19190433 100644 --- a/icechunk-python/tests/test_zarr/test_group.py +++ b/icechunk-python/tests/test_zarr/test_group.py @@ -12,6 +12,7 @@ from icechunk import IcechunkStore from tests.conftest import parse_repo from zarr import Array, AsyncArray, AsyncGroup, Group +from zarr.api.asynchronous import create_array from zarr.core.buffer import default_buffer_prototype from zarr.core.common import JSON, ZarrFormat from zarr.core.group import GroupMetadata @@ -28,7 +29,7 @@ def store(request: pytest.FixtureRequest) -> IcechunkStore: @pytest.fixture(params=[True, False]) -def exists_ok(request: pytest.FixtureRequest) -> bool: +def overwrite(request: pytest.FixtureRequest) -> bool: result = request.param if not isinstance(result, bool): raise TypeError("Wrong type returned by test fixture.") @@ -117,7 +118,7 @@ def test_group_members(store: IcechunkStore, zarr_format: ZarrFormat) -> None: subsubsubgroup = subsubgroup.create_group("subsubsubgroup") members_expected["subarray"] = group.create_array( - "subarray", shape=(100,), dtype="uint8", chunk_shape=(10,), exists_ok=True + "subarray", shape=(100,), dtype="uint8", chunks=(10,), overwrite=True ) # This is not supported by Icechunk, so we expect an error @@ -181,7 +182,7 @@ def test_group(store: IcechunkStore, zarr_format: ZarrFormat) -> None: # create an array from the "bar" group data = np.arange(0, 4 * 4, dtype="uint16").reshape((4, 4)) arr = bar.create_array( - "baz", shape=data.shape, dtype=data.dtype, chunk_shape=(2, 2), exists_ok=True + "baz", shape=data.shape, dtype=data.dtype, chunks=(2, 2), overwrite=True ) arr[:] = data @@ -207,25 +208,25 @@ def test_group(store: IcechunkStore, zarr_format: ZarrFormat) -> None: def test_group_create( - store: IcechunkStore, exists_ok: bool, zarr_format: ZarrFormat + store: IcechunkStore, overwrite: bool, zarr_format: ZarrFormat ) -> None: """ Test that `Group.create` works as expected. """ attributes = {"foo": 100} group = Group.from_store( - store, attributes=attributes, zarr_format=zarr_format, exists_ok=exists_ok + store, attributes=attributes, zarr_format=zarr_format, overwrite=overwrite ) assert group.attrs == attributes - if not exists_ok: + if not overwrite: with pytest.raises(ContainsGroupError): - _ = Group.from_store(store, exists_ok=exists_ok, zarr_format=zarr_format) + _ = Group.from_store(store, overwrite=overwrite, zarr_format=zarr_format) def test_group_open( - store: IcechunkStore, zarr_format: ZarrFormat, exists_ok: bool + store: IcechunkStore, zarr_format: ZarrFormat, overwrite: bool ) -> None: """ Test the `Group.open` method. @@ -238,26 +239,26 @@ def test_group_open( # create the group attrs = {"path": "foo"} group_created = Group.from_store( - store, attributes=attrs, zarr_format=zarr_format, exists_ok=exists_ok + store, attributes=attrs, zarr_format=zarr_format, overwrite=overwrite ) assert group_created.attrs == attrs assert group_created.metadata.zarr_format == zarr_format assert group_created.store_path == spath - # attempt to create a new group in place, to test exists_ok + # attempt to create a new group in place, to test overwrite new_attrs = {"path": "bar"} - if not exists_ok: + if not overwrite: with pytest.raises(ContainsGroupError): Group.from_store( - store, attributes=attrs, zarr_format=zarr_format, exists_ok=exists_ok + store, attributes=attrs, zarr_format=zarr_format, overwrite=overwrite ) else: if not store.supports_deletes: pytest.skip( - "Store does not support deletes but `exists_ok` is True, requiring deletes to override a group" + "Store does not support deletes but `overwrite` is True, requiring deletes to override a group" ) group_created_again = Group.from_store( - store, attributes=new_attrs, zarr_format=zarr_format, exists_ok=exists_ok + store, attributes=new_attrs, zarr_format=zarr_format, overwrite=overwrite ) assert group_created_again.attrs == new_attrs assert group_created_again.metadata.zarr_format == zarr_format @@ -271,7 +272,7 @@ def test_group_getitem(store: IcechunkStore, zarr_format: ZarrFormat) -> None: group = Group.from_store(store, zarr_format=zarr_format) subgroup = group.create_group(name="subgroup") - subarray = group.create_array(name="subarray", shape=(10,), chunk_shape=(10,)) + subarray = group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") assert group["subgroup"] == subgroup assert group["subarray"] == subarray @@ -286,7 +287,7 @@ def test_group_delitem(store: IcechunkStore, zarr_format: ZarrFormat) -> None: group = Group.from_store(store, zarr_format=zarr_format) subgroup = group.create_group(name="subgroup") - subarray = group.create_array(name="subarray", shape=(10,), chunk_shape=(10,)) + subarray = group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") assert group["subgroup"] == subgroup assert group["subarray"] == subarray @@ -347,7 +348,7 @@ def test_group_subgroups(store: IcechunkStore, zarr_format: ZarrFormat) -> None: keys = ("foo", "bar") subgroups_expected = tuple(group.create_group(k) for k in keys) # create a sub-array as well - _ = group.create_array("array", shape=(10,)) + _ = group.create_array("array", shape=(10,), dtype="uint8") subgroups_observed = tuple(a[1] for a in group.groups()) assert set(group.group_keys()) == set(keys) assert len(subgroups_observed) == len(subgroups_expected) @@ -360,7 +361,7 @@ def test_group_subarrays(store: IcechunkStore, zarr_format: ZarrFormat) -> None: """ group = Group.from_store(store, zarr_format=zarr_format) keys = ("foo", "bar") - subarrays_expected = tuple(group.create_array(k, shape=(10,)) for k in keys) + subarrays_expected = tuple(group.create_array(k, shape=(10,), dtype="uint8") for k in keys) # create a sub-group as well _ = group.create_group("group") subarrays_observed = tuple(a[1] for a in group.arrays()) @@ -399,7 +400,7 @@ async def test_group_update_attributes_async( def test_group_create_array( store: IcechunkStore, zarr_format: ZarrFormat, - exists_ok: bool, + overwrite: bool, method: Literal["create_array", "array"], ) -> None: """ @@ -411,20 +412,24 @@ def test_group_create_array( data = np.arange(np.prod(shape)).reshape(shape).astype(dtype) if method == "create_array": - array = group.create_array(name="array", shape=shape, dtype=dtype, data=data) + array = group.create_array(name="array", shape=shape, dtype=dtype) + array[:] = data elif method == "array": with pytest.warns(DeprecationWarning): - array = group.array(name="array", shape=shape, dtype=dtype, data=data) + array = group.array(name="array", shape=shape, dtype=dtype) + array[:] = data else: raise AssertionError - if not exists_ok: + if not overwrite: if method == "create_array": with pytest.raises(ContainsArrayError): - group.create_array(name="array", shape=shape, dtype=dtype, data=data) + array = group.create_array(name="array", shape=shape, dtype=dtype) + array[:] = data elif method == "array": with pytest.raises(ContainsArrayError), pytest.warns(DeprecationWarning): - group.array(name="array", shape=shape, dtype=dtype, data=data) + array = group.array(name="array", shape=shape, dtype=dtype) + array[:] = data assert array.shape == shape assert array.dtype == np.dtype(dtype) assert np.array_equal(array[:], data) @@ -432,12 +437,12 @@ def test_group_create_array( @pytest.mark.parametrize("store", ["memory"], indirect=["store"]) @pytest.mark.parametrize("zarr_format", [3]) -@pytest.mark.parametrize("exists_ok", [True, False]) +@pytest.mark.parametrize("overwrite", [True, False]) @pytest.mark.parametrize("extant_node", ["array", "group"]) def test_group_creation_existing_node( store: IcechunkStore, zarr_format: ZarrFormat, - exists_ok: bool, + overwrite: bool, extant_node: Literal["array", "group"], ) -> None: """ @@ -461,13 +466,13 @@ def test_group_creation_existing_node( new_attributes = {"new": True} - if exists_ok: - pytest.xfail("exists_ok is not implemented for Group.from_store") + if overwrite: + pytest.xfail("overwrite is not implemented for Group.from_store") # node_new = Group.from_store( # spath / "extant", # attributes=new_attributes, # zarr_format=zarr_format, - # exists_ok=exists_ok, + # overwrite=overwrite, # ) # assert node_new.attrs == new_attributes else: @@ -476,13 +481,13 @@ def test_group_creation_existing_node( spath / "extant", attributes=new_attributes, zarr_format=zarr_format, - exists_ok=exists_ok, + overwrite=overwrite, ) async def test_asyncgroup_create( store: IcechunkStore, - exists_ok: bool, + overwrite: bool, zarr_format: ZarrFormat, ) -> None: """ @@ -493,7 +498,7 @@ async def test_asyncgroup_create( agroup = await AsyncGroup.from_store( store, attributes=attributes, - exists_ok=exists_ok, + overwrite=overwrite, zarr_format=zarr_format, ) @@ -502,24 +507,24 @@ async def test_asyncgroup_create( ) assert agroup.store_path == await make_store_path(store) - if not exists_ok: + if not overwrite: with pytest.raises(ContainsGroupError): agroup = await AsyncGroup.from_store( spath, attributes=attributes, - exists_ok=exists_ok, + overwrite=overwrite, zarr_format=zarr_format, ) # create an array at our target path collision_name = "foo" - _ = await AsyncArray.create( + _ = await create_array( spath / collision_name, shape=(10,), dtype="uint8", zarr_format=zarr_format ) with pytest.raises(ContainsArrayError): _ = await AsyncGroup.from_store( StorePath(store=store) / collision_name, attributes=attributes, - exists_ok=exists_ok, + overwrite=overwrite, zarr_format=zarr_format, ) @@ -534,7 +539,7 @@ async def test_asyncgroup_delitem(store: IcechunkStore, zarr_format: ZarrFormat) name=array_name, shape=(10,), dtype="uint8", - chunk_shape=(2,), + chunks=(2,), attributes={"foo": 100}, ) await agroup.delitem(array_name) @@ -581,11 +586,11 @@ async def test_group_members_async(store: IcechunkStore) -> None: GroupMetadata(), store_path=StorePath(store=store, path="root"), ) - a0 = await group.create_array("a0", shape=(1,)) + a0 = await group.create_array("a0", shape=(1,), dtype="uint8") g0 = await group.create_group("g0") - a1 = await g0.create_array("a1", shape=(1,)) + a1 = await g0.create_array("a1", shape=(1,), dtype="uint8") g1 = await g0.create_group("g1") - a2 = await g1.create_array("a2", shape=(1,)) + a2 = await g1.create_array("a2", shape=(1,), dtype="uint8") g2 = await g1.create_group("g2") # immediate children @@ -650,7 +655,7 @@ def test_delitem_removes_children(store: IcechunkStore, zarr_format: ZarrFormat) g1 = zarr.group(store=store, zarr_format=zarr_format) g1.create_group("0") g1.create_group("0/0") - arr = g1.create_array("0/0/0", shape=(1,)) + arr = g1.create_array("0/0/0", shape=(1,), dtype="uint8") arr[:] = 1 del g1["0"] From dde37a1cfd74f68842b8b56a291ec8eb1a699fd2 Mon Sep 17 00:00:00 2001 From: Matthew Iannucci Date: Mon, 6 Jan 2025 09:26:23 -0500 Subject: [PATCH 03/16] Working through updates --- icechunk-python/notebooks/demo-dummy-data.ipynb | 2 +- icechunk-python/notebooks/demo-s3.ipynb | 2 +- icechunk-python/tests/test_store.py | 2 +- icechunk-python/tests/test_virtual_ref.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/icechunk-python/notebooks/demo-dummy-data.ipynb b/icechunk-python/notebooks/demo-dummy-data.ipynb index 8929809c..31bbb80c 100644 --- a/icechunk-python/notebooks/demo-dummy-data.ipynb +++ b/icechunk-python/notebooks/demo-dummy-data.ipynb @@ -125,7 +125,7 @@ " dimension_names=dims,\n", " attributes=attrs,\n", " data=array,\n", - " exists_ok=True,\n", + " overwrite=True,\n", " )\n", "\n", " return array" diff --git a/icechunk-python/notebooks/demo-s3.ipynb b/icechunk-python/notebooks/demo-s3.ipynb index 7777eebc..3d24476c 100644 --- a/icechunk-python/notebooks/demo-s3.ipynb +++ b/icechunk-python/notebooks/demo-s3.ipynb @@ -142,7 +142,7 @@ " fill_value=-1234567,\n", " dtype=oscar[var].dtype,\n", " data=oscar[var],\n", - " exists_ok=True,\n", + " overwrite=True,\n", " )\n", " print(session.commit(f\"wrote {var}\"))\n", " print(f\"committed; {time.time() - tic} seconds\")" diff --git a/icechunk-python/tests/test_store.py b/icechunk-python/tests/test_store.py index ee483ee7..9fc5a7ec 100644 --- a/icechunk-python/tests/test_store.py +++ b/icechunk-python/tests/test_store.py @@ -38,7 +38,7 @@ async def test_store_clear_chunk_list() -> None: await store.clear() group = zarr.group(store=store) - array = group.create_array(**array_kwargs, exists_ok=True) + array = group.create_array(**array_kwargs, overwrite=True) assert len([_ async for _ in store.list_prefix("/")]) == 2 array[:] = rng.integers( low=0, diff --git a/icechunk-python/tests/test_virtual_ref.py b/icechunk-python/tests/test_virtual_ref.py index fc097c46..075a9517 100644 --- a/icechunk-python/tests/test_virtual_ref.py +++ b/icechunk-python/tests/test_virtual_ref.py @@ -56,7 +56,7 @@ async def test_write_minio_virtual_refs() -> None: session = repo.writable_session("main") store = session.store - array = zarr.Array.create(store, shape=(5, 1, 3), chunks=(1, 1, 1), dtype="i4") + array = zarr.create_array(store, shape=(5, 1, 3), chunks=(1, 1, 1), dtype="i4") # We add the virtual chunk refs without checksum, with the right etag, and with the wrong wrong etag and datetime. # This way we can check retrieval operations that should fail From 19a5bbb9063b31634a76f1056baa5cb12479f640 Mon Sep 17 00:00:00 2001 From: Matthew Iannucci Date: Mon, 6 Jan 2025 09:51:03 -0500 Subject: [PATCH 04/16] More updates --- icechunk-python/pyproject.toml | 2 +- icechunk-python/tests/test_virtual_ref.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/icechunk-python/pyproject.toml b/icechunk-python/pyproject.toml index a38b5f24..99a1c1f6 100644 --- a/icechunk-python/pyproject.toml +++ b/icechunk-python/pyproject.toml @@ -39,7 +39,7 @@ test = [ "ruff", "dask>=2024.11.0", "distributed>=2024.11.0", - "xarray>=2024.11.0, <2025", + "xarray>=2025", "hypothesis", "pandas-stubs", "boto3-stubs[s3]", diff --git a/icechunk-python/tests/test_virtual_ref.py b/icechunk-python/tests/test_virtual_ref.py index 075a9517..12934ab8 100644 --- a/icechunk-python/tests/test_virtual_ref.py +++ b/icechunk-python/tests/test_virtual_ref.py @@ -56,7 +56,9 @@ async def test_write_minio_virtual_refs() -> None: session = repo.writable_session("main") store = session.store - array = zarr.create_array(store, shape=(5, 1, 3), chunks=(1, 1, 1), dtype="i4") + array = zarr.create_array( + store, shape=(5, 1, 3), chunks=(1, 1, 1), dtype="i4", compressors=None + ) # We add the virtual chunk refs without checksum, with the right etag, and with the wrong wrong etag and datetime. # This way we can check retrieval operations that should fail @@ -198,7 +200,7 @@ async def test_from_s3_public_virtual_refs(tmpdir: Path) -> None: root = zarr.Group.from_store(store=store, zarr_format=3) year = root.require_array( - name="year", shape=((72,)), chunks=((72,)), dtype="float32" + name="year", shape=((72,)), chunks=((72,)), dtype="float32", compressors=None ) store.set_virtual_ref( From 4ff77550cb8c04a8594316a72f4742aa37ffd5b5 Mon Sep 17 00:00:00 2001 From: Matthew Iannucci Date: Mon, 6 Jan 2025 11:30:12 -0500 Subject: [PATCH 05/16] Update to latest api --- icechunk-python/tests/test_config.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/icechunk-python/tests/test_config.py b/icechunk-python/tests/test_config.py index aa6fcf4a..056ad8e8 100644 --- a/icechunk-python/tests/test_config.py +++ b/icechunk-python/tests/test_config.py @@ -27,13 +27,12 @@ def tmp_store(tmpdir: Path) -> Generator[tuple[icechunk.IcechunkStore, str]]: def test_no_inline_chunks(tmp_store: tuple[icechunk.IcechunkStore, str]) -> None: store = tmp_store[0] store_path = tmp_store[1] - array = zarr.open_array( + array = zarr.create_array( store=store, - mode="a", shape=(10), dtype="int64", zarr_format=3, - chunks=(1), + chunks=(1,), fill_value=-1, ) array[:] = 42 @@ -48,14 +47,13 @@ def test_inline_chunks(tmp_store: tuple[icechunk.IcechunkStore, str]) -> None: store = tmp_store[0] store_path = tmp_store[1] - inline_array = zarr.open_array( + inline_array = zarr.create_array( store=store, - mode="a", - path="inline", + name="inline", shape=(10), dtype="int32", zarr_format=3, - chunks=(1), + chunks=(1,), fill_value=-1, ) @@ -65,14 +63,13 @@ def test_inline_chunks(tmp_store: tuple[icechunk.IcechunkStore, str]) -> None: # inline_chunk_threshold is 40, we should have no chunks directory assert not os.path.isdir(f"{store_path}/chunks") - written_array = zarr.open_array( + written_array = zarr.create_array( store=store, - mode="a", - path="not_inline", + name="not_inline", shape=(10), dtype="int64", zarr_format=3, - chunks=(1), + chunks=(1,), fill_value=-1, ) From 6b59fa2143cdd48ef834286933a74c1fde23b6c7 Mon Sep 17 00:00:00 2001 From: Matthew Iannucci Date: Mon, 6 Jan 2025 11:40:58 -0500 Subject: [PATCH 06/16] Fix config issue with zstd --- icechunk-python/tests/test_config.py | 2 ++ icechunk-python/tests/test_conflicts.py | 4 +--- icechunk-python/tests/test_zarr/test_array.py | 2 +- icechunk-python/tests/test_zarr/test_group.py | 14 ++++++++++---- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/icechunk-python/tests/test_config.py b/icechunk-python/tests/test_config.py index 056ad8e8..7495d5bf 100644 --- a/icechunk-python/tests/test_config.py +++ b/icechunk-python/tests/test_config.py @@ -55,6 +55,7 @@ def test_inline_chunks(tmp_store: tuple[icechunk.IcechunkStore, str]) -> None: zarr_format=3, chunks=(1,), fill_value=-1, + compressors=None, ) inline_array[:] = 9 @@ -71,6 +72,7 @@ def test_inline_chunks(tmp_store: tuple[icechunk.IcechunkStore, str]) -> None: zarr_format=3, chunks=(1,), fill_value=-1, + compressors=None, ) written_array[:] = 3 diff --git a/icechunk-python/tests/test_conflicts.py b/icechunk-python/tests/test_conflicts.py index 6f62b30d..1f7c4ded 100644 --- a/icechunk-python/tests/test_conflicts.py +++ b/icechunk-python/tests/test_conflicts.py @@ -18,9 +18,7 @@ def repo(tmpdir: Path) -> icechunk.Repository: store = session.store root = zarr.group(store=store) root.create_group("foo/bar") - root.create_array( - "foo/bar/some-array", shape=(10, 10), chunks=(1, 1), dtype="i4" - ) + root.create_array("foo/bar/some-array", shape=(10, 10), chunks=(1, 1), dtype="i4") session.commit("commit 1") return repo diff --git a/icechunk-python/tests/test_zarr/test_array.py b/icechunk-python/tests/test_zarr/test_array.py index c04b2da6..0cd28a0c 100644 --- a/icechunk-python/tests/test_zarr/test_array.py +++ b/icechunk-python/tests/test_zarr/test_array.py @@ -10,7 +10,7 @@ from icechunk import IcechunkStore from tests.conftest import parse_repo -from zarr import Array, Group, create_array +from zarr import Group, create_array from zarr.core.buffer import default_buffer_prototype from zarr.core.common import ZarrFormat from zarr.errors import ContainsArrayError, ContainsGroupError diff --git a/icechunk-python/tests/test_zarr/test_group.py b/icechunk-python/tests/test_zarr/test_group.py index 19190433..1fd35a04 100644 --- a/icechunk-python/tests/test_zarr/test_group.py +++ b/icechunk-python/tests/test_zarr/test_group.py @@ -11,7 +11,7 @@ import zarr.api.asynchronous from icechunk import IcechunkStore from tests.conftest import parse_repo -from zarr import Array, AsyncArray, AsyncGroup, Group +from zarr import Array, AsyncGroup, Group from zarr.api.asynchronous import create_array from zarr.core.buffer import default_buffer_prototype from zarr.core.common import JSON, ZarrFormat @@ -272,7 +272,9 @@ def test_group_getitem(store: IcechunkStore, zarr_format: ZarrFormat) -> None: group = Group.from_store(store, zarr_format=zarr_format) subgroup = group.create_group(name="subgroup") - subarray = group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") + subarray = group.create_array( + name="subarray", shape=(10,), chunks=(10,), dtype="uint8" + ) assert group["subgroup"] == subgroup assert group["subarray"] == subarray @@ -287,7 +289,9 @@ def test_group_delitem(store: IcechunkStore, zarr_format: ZarrFormat) -> None: group = Group.from_store(store, zarr_format=zarr_format) subgroup = group.create_group(name="subgroup") - subarray = group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") + subarray = group.create_array( + name="subarray", shape=(10,), chunks=(10,), dtype="uint8" + ) assert group["subgroup"] == subgroup assert group["subarray"] == subarray @@ -361,7 +365,9 @@ def test_group_subarrays(store: IcechunkStore, zarr_format: ZarrFormat) -> None: """ group = Group.from_store(store, zarr_format=zarr_format) keys = ("foo", "bar") - subarrays_expected = tuple(group.create_array(k, shape=(10,), dtype="uint8") for k in keys) + subarrays_expected = tuple( + group.create_array(k, shape=(10,), dtype="uint8") for k in keys + ) # create a sub-group as well _ = group.create_group("group") subarrays_observed = tuple(a[1] for a in group.arrays()) From 13d5f344bfa71d05a1977b888f1b7d67f8a90350 Mon Sep 17 00:00:00 2001 From: Matthew Iannucci Date: Tue, 7 Jan 2025 09:30:43 -0500 Subject: [PATCH 07/16] Update xarray for now, still failing --- icechunk-python/pyproject.toml | 4 +++- icechunk-python/tests/test_dask.py | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/icechunk-python/pyproject.toml b/icechunk-python/pyproject.toml index 99a1c1f6..9bed5f29 100644 --- a/icechunk-python/pyproject.toml +++ b/icechunk-python/pyproject.toml @@ -39,7 +39,7 @@ test = [ "ruff", "dask>=2024.11.0", "distributed>=2024.11.0", - "xarray>=2025", + "xarray@git+https://github.com/pydata/xarray.git@main", "hypothesis", "pandas-stubs", "boto3-stubs[s3]", @@ -63,6 +63,8 @@ filterwarnings = [ "error", "ignore:Jupyter is migrating its paths to use:DeprecationWarning:", "ignore:Port 8787 is already in use:UserWarning:", + # TODO: this is raised for vlen-utf8, consolidated metadata, U1 dtype + "ignore:is currently not part .* the Zarr version 3 specification.", ] [tool.pyright] diff --git a/icechunk-python/tests/test_dask.py b/icechunk-python/tests/test_dask.py index d2432ccc..031e32f2 100644 --- a/icechunk-python/tests/test_dask.py +++ b/icechunk-python/tests/test_dask.py @@ -10,7 +10,6 @@ def test_distributed() -> None: - pytest.xfail() with distributed.Client(): # type: ignore [no-untyped-call] ds = create_test_data().chunk(dim1=3, dim2=4) with roundtrip(ds) as actual: @@ -22,7 +21,6 @@ def test_distributed() -> None: def test_threaded() -> None: - pytest.xfail() with dask.config.set(scheduler="threads"): ds = create_test_data().chunk(dim1=3, dim2=4) with roundtrip(ds) as actual: From 99c6e0f446fc4fee03ac38b54c24e264809e099c Mon Sep 17 00:00:00 2001 From: Matthew Iannucci Date: Tue, 7 Jan 2025 09:47:07 -0500 Subject: [PATCH 08/16] fix warnign filter --- icechunk-python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icechunk-python/pyproject.toml b/icechunk-python/pyproject.toml index 9bed5f29..84dc56dd 100644 --- a/icechunk-python/pyproject.toml +++ b/icechunk-python/pyproject.toml @@ -64,7 +64,7 @@ filterwarnings = [ "ignore:Jupyter is migrating its paths to use:DeprecationWarning:", "ignore:Port 8787 is already in use:UserWarning:", # TODO: this is raised for vlen-utf8, consolidated metadata, U1 dtype - "ignore:is currently not part .* the Zarr version 3 specification.", + "ignore:.*is currently not part .* the Zarr version 3 specification.", ] [tool.pyright] From 34141f5e4a92456b6cc46688f83909acfdbc890d Mon Sep 17 00:00:00 2001 From: Matthew Iannucci Date: Tue, 7 Jan 2025 12:07:04 -0500 Subject: [PATCH 09/16] Update icechunk-python/pyproject.toml Co-authored-by: Joe Hamman --- icechunk-python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icechunk-python/pyproject.toml b/icechunk-python/pyproject.toml index 84dc56dd..24f68722 100644 --- a/icechunk-python/pyproject.toml +++ b/icechunk-python/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ license = { text = "Apache-2.0" } dynamic = ["version"] -dependencies = ["zarr==3.0.0rc1"] +dependencies = ["zarr==3.0.0rc2"] [tool.poetry] name = "icechunk" From dff9b47ab1208fff098cfbfd364de68ba24f3170 Mon Sep 17 00:00:00 2001 From: Matthew Iannucci Date: Tue, 7 Jan 2025 13:33:44 -0500 Subject: [PATCH 10/16] I couldnt get wildcard to work, so just added both --- icechunk-python/pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/icechunk-python/pyproject.toml b/icechunk-python/pyproject.toml index 24f68722..43a6c404 100644 --- a/icechunk-python/pyproject.toml +++ b/icechunk-python/pyproject.toml @@ -64,7 +64,8 @@ filterwarnings = [ "ignore:Jupyter is migrating its paths to use:DeprecationWarning:", "ignore:Port 8787 is already in use:UserWarning:", # TODO: this is raised for vlen-utf8, consolidated metadata, U1 dtype - "ignore:.*is currently not part .* the Zarr version 3 specification.", + "ignore:The codec `vlen-utf8` is currently not part in the Zarr format 3 specification.", + "ignore:The dtype ` Date: Tue, 7 Jan 2025 13:36:14 -0500 Subject: [PATCH 11/16] Add back tests for overwrite --- icechunk-python/tests/test_zarr/test_array.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/icechunk-python/tests/test_zarr/test_array.py b/icechunk-python/tests/test_zarr/test_array.py index 0cd28a0c..69f30091 100644 --- a/icechunk-python/tests/test_zarr/test_array.py +++ b/icechunk-python/tests/test_zarr/test_array.py @@ -55,16 +55,15 @@ def test_array_creation_existing_node( if overwrite: # This is currently not supported by IcechunkStore - pytest.xfail("IcechunkStore does not support overwrite=True") - # arr_new = create_array( - # spath / "extant", - # shape=new_shape, - # dtype=new_dtype, - # overwrite=overwrite, - # zarr_format=zarr_format, - # ) - # assert arr_new.shape == new_shape - # assert arr_new.dtype == new_dtype + arr_new = create_array( + spath / "extant", + shape=new_shape, + dtype=new_dtype, + overwrite=overwrite, + zarr_format=zarr_format, + ) + assert arr_new.shape == new_shape + assert arr_new.dtype == new_dtype else: with pytest.raises(expected_exception): create_array( From cda79973478717642cd8c1338272b6943b2248df Mon Sep 17 00:00:00 2001 From: Matthew Iannucci Date: Tue, 7 Jan 2025 13:39:26 -0500 Subject: [PATCH 12/16] Test overwrite for groups --- icechunk-python/tests/test_zarr/test_group.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/icechunk-python/tests/test_zarr/test_group.py b/icechunk-python/tests/test_zarr/test_group.py index 1fd35a04..04e0e809 100644 --- a/icechunk-python/tests/test_zarr/test_group.py +++ b/icechunk-python/tests/test_zarr/test_group.py @@ -473,14 +473,13 @@ def test_group_creation_existing_node( new_attributes = {"new": True} if overwrite: - pytest.xfail("overwrite is not implemented for Group.from_store") - # node_new = Group.from_store( - # spath / "extant", - # attributes=new_attributes, - # zarr_format=zarr_format, - # overwrite=overwrite, - # ) - # assert node_new.attrs == new_attributes + node_new = Group.from_store( + spath / "extant", + attributes=new_attributes, + zarr_format=zarr_format, + overwrite=overwrite, + ) + assert node_new.attrs == new_attributes else: with pytest.raises(expected_exception): Group.from_store( From 935b89df8e377e0ad4312e665f756bd4b5d4cad6 Mon Sep 17 00:00:00 2001 From: Matthew Iannucci Date: Tue, 7 Jan 2025 13:44:25 -0500 Subject: [PATCH 13/16] Update backend test --- icechunk-python/tests/run_xarray_backends_tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/icechunk-python/tests/run_xarray_backends_tests.py b/icechunk-python/tests/run_xarray_backends_tests.py index c6963623..2d3b750b 100644 --- a/icechunk-python/tests/run_xarray_backends_tests.py +++ b/icechunk-python/tests/run_xarray_backends_tests.py @@ -47,7 +47,7 @@ def test_roundtrip_consolidated(self, consolidated: Any) -> None: class TestIcechunkStoreFilesystem(IcechunkStoreBase): @contextlib.contextmanager def create_zarr_target(self) -> Generator[IcechunkStore]: - if zarr.config.config["default_zarr_version"] == 2: + if zarr.config.config["default_zarr_format"] == 2: pytest.skip("v2 not supported") with tempfile.TemporaryDirectory() as tmpdir: repo = Repository.create(local_filesystem_storage(tmpdir)) @@ -58,7 +58,7 @@ def create_zarr_target(self) -> Generator[IcechunkStore]: class TestIcechunkStoreMemory(IcechunkStoreBase): @contextlib.contextmanager def create_zarr_target(self) -> Generator[IcechunkStore]: - if zarr.config.config["default_zarr_version"] == 2: + if zarr.config.config["default_zarr_format"] == 2: pytest.skip("v2 not supported") repo = Repository.create(in_memory_storage()) session = repo.writable_session("main") @@ -74,7 +74,7 @@ def test_pickle_dataarray(self) -> None: class TestIcechunkStoreMinio(IcechunkStoreBase): @contextlib.contextmanager def create_zarr_target(self) -> Generator[IcechunkStore]: - if zarr.config.config["default_zarr_version"] == 2: + if zarr.config.config["default_zarr_format"] == 2: pytest.skip("v2 not supported") repo = Repository.create( s3_storage( From 6fb5d34d3c42949df91afd116e2adb0c1a69cdfa Mon Sep 17 00:00:00 2001 From: Matthew Iannucci Date: Tue, 7 Jan 2025 13:48:17 -0500 Subject: [PATCH 14/16] Fix --- icechunk-python/tests/run_xarray_backends_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icechunk-python/tests/run_xarray_backends_tests.py b/icechunk-python/tests/run_xarray_backends_tests.py index 2d3b750b..3998dd5e 100644 --- a/icechunk-python/tests/run_xarray_backends_tests.py +++ b/icechunk-python/tests/run_xarray_backends_tests.py @@ -17,7 +17,7 @@ ) from xarray.tests.test_backends import ( ZarrBase, - default_zarr_version, # noqa: F401; needed otherwise not discovered + default_zarr_format, # noqa: F401; needed otherwise not discovered ) From 4eb3c2f85634c26e8032f2add87feaedba2d6d10 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Tue, 7 Jan 2025 12:33:38 -0700 Subject: [PATCH 15/16] Add pip list step --- .github/workflows/python-check.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/python-check.yaml b/.github/workflows/python-check.yaml index 35da0bce..9fc13df4 100644 --- a/.github/workflows/python-check.yaml +++ b/.github/workflows/python-check.yaml @@ -95,6 +95,15 @@ jobs: restore-keys: | cache-hypothesis- + - name: describe environment + shell: bash + working-directory: icechunk-python + run: | + set -e + python3 -m venv .venv + source .venv/bin/activate + pip list + - name: pytest shell: bash working-directory: icechunk-python From a752d9d8f872ffcf827e35a9dac6b73493c9ee3a Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Tue, 7 Jan 2025 12:40:37 -0700 Subject: [PATCH 16/16] try ignoring warning --- icechunk-python/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/icechunk-python/pyproject.toml b/icechunk-python/pyproject.toml index 43a6c404..e4892fcf 100644 --- a/icechunk-python/pyproject.toml +++ b/icechunk-python/pyproject.toml @@ -66,6 +66,7 @@ filterwarnings = [ # TODO: this is raised for vlen-utf8, consolidated metadata, U1 dtype "ignore:The codec `vlen-utf8` is currently not part in the Zarr format 3 specification.", "ignore:The dtype `