S3 - in progress

dacapo/store/create_store.py

@@ -31,7 +31,7 @@ def create_config_store():
         db_name = options.mongo_db_name
         return MongoConfigStore(db_host, db_name)
     elif options.type == "files":
-        store_path = Path(options.runs_base_dir).expanduser()
+        store_path = Path(options.runs_base_dir)
         return FileConfigStore(store_path / "configs")
     else:
         raise ValueError(f"Unknown store type {options.type}")
@@ -62,7 +62,7 @@ def create_stats_store():
         db_name = options.mongo_db_name
         return MongoStatsStore(db_host, db_name)
     elif options.type == "files":
-        store_path = Path(options.runs_base_dir).expanduser()
+        store_path = Path(options.runs_base_dir)
         return FileStatsStore(store_path / "stats")
     else:
         raise ValueError(f"Unknown store type {options.type}")
@@ -85,8 +85,7 @@ def create_weights_store():
 
     options = Options.instance()
 
-    # currently, only the LocalWeightsStore is supported
-    base_dir = Path(options.runs_base_dir).expanduser()
+    base_dir = Path(options.runs_base_dir)
     return LocalWeightsStore(base_dir)
 
 

examples/aws/README.md

@@ -0,0 +1,14 @@
+You can work locally using S3 data by setting the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables. You can also set the `AWS_REGION` environment variable to specify the region to use. If you are using a profile, you can set the `AWS_PROFILE` environment variable to specify the profile to use.
+
+```bash
+aws configure
+```
+
+In order to store checkpoints and experiments data in S3, you need to modify `dacapo.yaml` to include the following:
+
+```yaml
+runs_base_dir: "s3://dacapotest"
+```
+
+For configs and stats, you can save them locally or s3 by setting `type: files` or for mongodb by setting `type: mongo` in the `dacapo.yaml` file.
+

examples/aws/aws_store_check.py

@@ -0,0 +1,30 @@
+# %%
+import dacapo
+
+# from  import create_config_store
+
+config_store = dacapo.store.create_store.create_config_store()
+
+# %%
+from dacapo import Options
+
+options = Options.instance()
+
+# %%
+options
+# %%
+from dacapo.experiments.tasks import DistanceTaskConfig
+
+task_config = DistanceTaskConfig(
+    name="cosem_distance_task_4nm",
+    channels=["mito"],
+    clip_distance=40.0,
+    tol_distance=40.0,
+    scale_factor=80.0,
+)
+
+# %%
+
+config_store.store_task_config(task_config)
+
+# %%

examples/aws/cloud_csv.csv

@@ -0,0 +1,3 @@
+train,s3://janelia-cosem-datasets/jrc_hela-2/jrc_hela-2.zarr,recon-1/em/fibsem-uint8,s3://janelia-cosem-datasets/jrc_hela-2/jrc_hela-2.zarr,recon-1/labels/groundtruth/crop155/[nuc]
+train,s3://janelia-cosem-datasets/jrc_hela-2/jrc_hela-2.zarr,recon-1/em/fibsem-uint8,s3://janelia-cosem-datasets/jrc_hela-2/jrc_hela-2.zarr,recon-1/labels/groundtruth/crop7/[nuc]
+val,s3://janelia-cosem-datasets/jrc_hela-2/jrc_hela-2.zarr,recon-1/em/fibsem-uint8,s3://janelia-cosem-datasets/jrc_hela-2/jrc_hela-2.zarr,recon-1/labels/groundtruth/crop6/[nuc]

examples/aws/dacapo.yaml

@@ -0,0 +1,3 @@
+
+runs_base_dir: "s3://dacapotest"
+type: "files"

examples/aws/s3_datasplit.py

@@ -0,0 +1,16 @@
+# %%
+from dacapo.experiments.datasplits import DataSplitGenerator
+from funlib.geometry import Coordinate
+
+input_resolution = Coordinate(8, 8, 8)
+output_resolution = Coordinate(4, 4, 4)
+datasplit_config = DataSplitGenerator.generate_from_csv(
+    "cloud_csv.csv",
+    input_resolution,
+    output_resolution,
+).compute()
+# %%
+datasplit = datasplit_config.datasplit_type(datasplit_config)
+# %%
+viewer = datasplit._neuroglancer()
+# %%

pyproject.toml

@@ -56,6 +56,8 @@ dependencies = [
         "click",
         "pyyaml",
         "scipy",
+        "upath",
+        "boto3",
         ]
 
 # extras