fix(tensorstore): Manually create zarrdir via fsspec

openclimatefix · Nov 12, 2024 · a605bd3 · a605bd3
1 parent a612e73
commit a605bd3
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/src/nwp_consumer/internal/entities/tensorstore.py b/src/nwp_consumer/internal/entities/tensorstore.py
@@ -143,10 +143,12 @@ def initialize_empty_store(
                         anon=False,
                         client_kwargs={
                             "region_name": os.getenv("AWS_REGION", "eu-west-1"),
+                            "endpoint_url": os.getenv("AWS_ENDPOINT_URL", None),
                         },
                     )
                     path = zarrdir + "/" + TensorStore.gen_store_filename(coords=coords)
-                    store = s3fs.mapping.S3Map(path, fs, check=True, create=True)
+                    fs.mkdirs(path=path, exists_ok=True)
+                    store = s3fs.mapping.S3Map(path, fs, check=False, create=True)
                 except Exception as e:
                     return Failure(OSError(
                         f"Unable to create file mapping for path '{path}'. "