pvdaq and fargate examples added--fargate not working yet

slacgismo · Jan 19, 2024 · 7ca689f · 7ca689f
1 parent a26b266
commit 7ca689f
Show file tree

Hide file tree

Showing 12 changed files with 1,038 additions and 352 deletions.
diff --git a/dask/fargate/demo.ipynb b/dask/fargate/demo.ipynb
diff --git a/sdt_dask/__init__.py b/sdt_dask/__init__.py
diff --git a/sdt_dask/clients/__init__.py b/sdt_dask/clients/__init__.py
diff --git a/dask/fargate/Dockerfile → sdt_dask/clients/aws/Dockerfile b/dask/fargate/Dockerfile → sdt_dask/clients/aws/Dockerfile
diff --git a/dask/fargate/readme.md → sdt_dask/clients/aws/README.md b/dask/fargate/readme.md → sdt_dask/clients/aws/README.md
@@ -43,7 +43,7 @@ If you have additional dependencies, install them as well (both Docker image and
 ## Docker Image
 
 1. Install your python requirements and all dependencies as steps in your dockerfile.  
-   Here's a sample dockerfile with solar-data-tools dependencies and all the packages listed above: [link](./Dockerfile)  
+   Here's a sample dockerfile with solar-data-tools dependencies and all the packages listed above: [link](Dockerfile)  
 2. Build a docker image
    1. Go to the directory where your Dockerfile is
    2. ```docker build -t <YOUR_IMAGE_NAME> .```
@@ -84,7 +84,7 @@ This refers to the local environment where users run their python scripts to cre
 
 Now, let's try the demo!
 # Run Demo
-1. Get the demo script [here](./demo.ipynb)
+1. Get the demo script [here](../fargate/fargate_demo.ipynb)
 2. Open Jupyter Notebook, set the python interpreter (in the drop down list, choose the environment we just created)
 3. **Please add your stuff before using this script (see comments for details)**
 4. Run the script and wait for cluster initialization, you can check the scheduler and workers status via AWS ECS console

diff --git a/sdt_dask/clients/aws/__init__.py b/sdt_dask/clients/aws/__init__.py
diff --git a/sdt_dask/clients/aws/fargate.py b/sdt_dask/clients/aws/fargate.py
@@ -0,0 +1,27 @@
+from dask.distributed import Client
+from dask_cloudprovider.aws import FargateCluster
+r'''
+Optional:
+    tags:  only use this if your organization enforces tag policies
+Required:
+    image:   should be a dockerhub public image. Please customize your image if needed
+'''
+
+TAGS = {
+    "project-pa-number": "21691-H2001",
+    "project": "pvinsight"
+}
+
+IMAGE = "smiskov/dask-sdt-sm:latest"
+def _init_fargate_cluster(tags: dict, image: str, scale_num: int) -> FargateCluster:
+    cluster = FargateCluster(tags=tags, image=image)
+    cluster.scale(scale_num) # TODO: this probably should be accessible somewhere else
+    return cluster
+
+def _init_dask_client(cluster: FargateCluster) -> Client:
+    client = Client(cluster)
+    return client
+
+def get_fargate_cluster(tags=TAGS, image=IMAGE, scale_num=12) -> Client:
+    cluster = _init_fargate_cluster(tags, image, scale_num)
+    return _init_dask_client(cluster)
diff --git a/sdt_dask/dataplugs/__init__.py b/sdt_dask/dataplugs/__init__.py