From a3bd21d7631f4fb13d435a2dec9a7092568674eb Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Fri, 15 Nov 2024 11:18:01 +0100 Subject: [PATCH 01/20] Adjust dreambooth --- 06_gpu_and_ml/dreambooth/dreambooth_app.py | 11 +++++----- 10_integrations/streamlit/serve_streamlit.py | 22 +++++++------------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/06_gpu_and_ml/dreambooth/dreambooth_app.py b/06_gpu_and_ml/dreambooth/dreambooth_app.py index ddf1a818a..b6c1d3e9b 100644 --- a/06_gpu_and_ml/dreambooth/dreambooth_app.py +++ b/06_gpu_and_ml/dreambooth/dreambooth_app.py @@ -141,7 +141,6 @@ class SharedConfig: {"HF_HUB_ENABLE_HF_TRANSFER": "1"} # turn on faster downloads from HF ) - @app.function( volumes={MODEL_DIR: volume}, image=image, @@ -416,14 +415,16 @@ class AppConfig(SharedConfig): guidance_scale: float = 6 -assets_path = Path(__file__).parent / "assets" - +# attach local web assets +image_with_assets = image.attach_local_dir( + Path(__file__).parent / "assets", + remote_path="/assets" +) @app.function( - image=image, + image=image_with_assets, concurrency_limit=1, allow_concurrent_inputs=1000, - mounts=[modal.Mount.from_local_dir(assets_path, remote_path="/assets")], ) @modal.asgi_app() def fastapi_app(): diff --git a/10_integrations/streamlit/serve_streamlit.py b/10_integrations/streamlit/serve_streamlit.py index 37544a789..f837567e3 100644 --- a/10_integrations/streamlit/serve_streamlit.py +++ b/10_integrations/streamlit/serve_streamlit.py @@ -24,32 +24,25 @@ # ## Define container dependencies # # The `app.py` script imports three third-party packages, so we include these in the example's -# image definition. +# image definition and then attach the app.py file itself to the image + +streamlit_script_local_path = Path(__file__).parent / "app.py" +streamlit_script_remote_path = Path("/root/app.py") image = modal.Image.debian_slim(python_version="3.11").pip_install( "streamlit~=1.35.0", "numpy~=1.26.4", "pandas~=2.2.2" +).attach_local_file( + streamlit_script_local_path, + streamlit_script_remote_path, ) app = modal.App(name="example-modal-streamlit", image=image) -# ## Mounting the `app.py` script -# -# We can just mount the `app.py` script inside the container at a pre-defined path using a Modal -# [`Mount`](https://modal.com/docs/guide/local-data#mounting-directories). - -streamlit_script_local_path = Path(__file__).parent / "app.py" -streamlit_script_remote_path = Path("/root/app.py") - if not streamlit_script_local_path.exists(): raise RuntimeError( "app.py not found! Place the script with your streamlit app in the same directory." ) -streamlit_script_mount = modal.Mount.from_local_file( - streamlit_script_local_path, - streamlit_script_remote_path, -) - # ## Spawning the Streamlit server # # Inside the container, we will run the Streamlit server in a background subprocess using @@ -58,7 +51,6 @@ @app.function( allow_concurrent_inputs=100, - mounts=[streamlit_script_mount], ) @modal.web_server(8000) def run(): From 0910283d551f7c412d5a1da57aab23131fdb1a82 Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Fri, 15 Nov 2024 11:18:24 +0100 Subject: [PATCH 02/20] Adjust comfy --- 06_gpu_and_ml/comfyui/comfyapp.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/06_gpu_and_ml/comfyui/comfyapp.py b/06_gpu_and_ml/comfyui/comfyapp.py index d8c539362..9756e6c28 100644 --- a/06_gpu_and_ml/comfyui/comfyapp.py +++ b/06_gpu_and_ml/comfyui/comfyapp.py @@ -75,6 +75,11 @@ .run_commands( # use comfy-cli to install the ComfyUI repo and its dependencies "comfy --skip-prompt install --nvidia" ) + .attach_local_file( + Path(__file__).parent / "workflow_api.json", + "/root/workflow_api.json", + copy=True, + ) ) # #### Downloading models @@ -172,13 +177,7 @@ def ui(): @app.cls( allow_concurrent_inputs=10, container_idle_timeout=300, - gpu="A10G", - mounts=[ - modal.Mount.from_local_file( - Path(__file__).parent / "workflow_api.json", - "/root/workflow_api.json", - ), - ], + gpu="A10G" ) class ComfyUI: @modal.enter() From e89104646bbaa3183c4202393e71c511f40b4ba7 Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Tue, 26 Nov 2024 13:01:32 +0100 Subject: [PATCH 03/20] Add --- 06_gpu_and_ml/comfyui/comfyapp.py | 2 +- 06_gpu_and_ml/dreambooth/dreambooth_app.py | 2 +- 10_integrations/streamlit/serve_streamlit.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/06_gpu_and_ml/comfyui/comfyapp.py b/06_gpu_and_ml/comfyui/comfyapp.py index 9756e6c28..ee4b96ebf 100644 --- a/06_gpu_and_ml/comfyui/comfyapp.py +++ b/06_gpu_and_ml/comfyui/comfyapp.py @@ -75,7 +75,7 @@ .run_commands( # use comfy-cli to install the ComfyUI repo and its dependencies "comfy --skip-prompt install --nvidia" ) - .attach_local_file( + .add_local_file( Path(__file__).parent / "workflow_api.json", "/root/workflow_api.json", copy=True, diff --git a/06_gpu_and_ml/dreambooth/dreambooth_app.py b/06_gpu_and_ml/dreambooth/dreambooth_app.py index b6c1d3e9b..c92e36b1e 100644 --- a/06_gpu_and_ml/dreambooth/dreambooth_app.py +++ b/06_gpu_and_ml/dreambooth/dreambooth_app.py @@ -416,7 +416,7 @@ class AppConfig(SharedConfig): # attach local web assets -image_with_assets = image.attach_local_dir( +image_with_assets = image.add_local_dir( Path(__file__).parent / "assets", remote_path="/assets" ) diff --git a/10_integrations/streamlit/serve_streamlit.py b/10_integrations/streamlit/serve_streamlit.py index f837567e3..38bc6331b 100644 --- a/10_integrations/streamlit/serve_streamlit.py +++ b/10_integrations/streamlit/serve_streamlit.py @@ -31,7 +31,7 @@ image = modal.Image.debian_slim(python_version="3.11").pip_install( "streamlit~=1.35.0", "numpy~=1.26.4", "pandas~=2.2.2" -).attach_local_file( +).add_local_file( streamlit_script_local_path, streamlit_script_remote_path, ) From 80d067f425fc22e594e1f8516deee683995c061f Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Tue, 26 Nov 2024 13:05:59 +0100 Subject: [PATCH 04/20] Remove mounts from ocr webapp --- 09_job_queues/doc_ocr_webapp.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/09_job_queues/doc_ocr_webapp.py b/09_job_queues/doc_ocr_webapp.py index 3b8b90c7e..773664442 100644 --- a/09_job_queues/doc_ocr_webapp.py +++ b/09_job_queues/doc_ocr_webapp.py @@ -84,10 +84,8 @@ async def poll_results(call_id: str): assets_path = Path(__file__).parent / "doc_ocr_frontend" - @app.function( - image=modal.Image.debian_slim().pip_install("fastapi[standard]==0.115.4"), - mounts=[modal.Mount.from_local_dir(assets_path, remote_path="/assets")], + image=modal.Image.debian_slim().pip_install("fastapi[standard]==0.115.4").add_local_dir(assets_path, remote_path="/assets"), ) @modal.asgi_app() def wrapper(): From e651d6ea2e01ae42a94acc6a24be3611794532c3 Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Tue, 26 Nov 2024 13:06:11 +0100 Subject: [PATCH 05/20] Random type fix - Path to PosixPath --- 10_integrations/s3_bucket_mount.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/10_integrations/s3_bucket_mount.py b/10_integrations/s3_bucket_mount.py index e58d739db..dcf3bc2a8 100644 --- a/10_integrations/s3_bucket_mount.py +++ b/10_integrations/s3_bucket_mount.py @@ -24,7 +24,7 @@ # the relevant AWS credentials with your Modal apps. from datetime import datetime -from pathlib import Path +from pathlib import Path, PosixPath import modal @@ -38,8 +38,8 @@ required_keys=["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"], ) -MOUNT_PATH: Path = Path("/bucket") -YELLOW_TAXI_DATA_PATH: Path = MOUNT_PATH / "yellow_taxi" +MOUNT_PATH = PosixPath("/bucket") +YELLOW_TAXI_DATA_PATH = MOUNT_PATH / "yellow_taxi" # The dependencies installed above are not available locally. The following block instructs Modal From 509a55c4e23ff8c2b9f91740ddda1054610172e4 Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Tue, 26 Nov 2024 13:11:11 +0100 Subject: [PATCH 06/20] Docs --- 09_job_queues/doc_ocr_webapp.py | 17 ++++++++++------- .../dbt_modal_inference/dbt_modal_inference.py | 7 ++++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/09_job_queues/doc_ocr_webapp.py b/09_job_queues/doc_ocr_webapp.py index 773664442..152f658c4 100644 --- a/09_job_queues/doc_ocr_webapp.py +++ b/09_job_queues/doc_ocr_webapp.py @@ -74,25 +74,28 @@ async def poll_results(call_id: str): return result +# Specify a container image containing the version of fastapi we want to use to serve +# our web app +fast_api_image = modal.Image.debian_slim().pip_install("fastapi[standard]==0.115.4") -# Finally, we mount the static files for our front-end. We've made [a simple React +# Finally, we add the static files for our front-end. We've made [a simple React # app](https://github.com/modal-labs/modal-examples/tree/main/09_job_queues/doc_ocr_frontend) -# that hits the two endpoints defined above. To package these files with our app, first -# we get the local assets path, and then create a modal [`Mount`](https://modal.com/docs/guide/local-data#mounting-directories) -# that mounts this directory at `/assets` inside our container. Then, we instruct FastAPI to [serve +# that hits the two endpoints defined above. To package these files with our app, we use +# add_local_dir with the local directory of the assets, and specify that we want them +# in the `/assets` directory inside our container (the `remote_path`). Then, we instruct FastAPI to [serve # this static file directory](https://fastapi.tiangolo.com/tutorial/static-files/) at our root path. -assets_path = Path(__file__).parent / "doc_ocr_frontend" +local_assets_path = Path(__file__).parent / "doc_ocr_frontend" +ocr_app_image = fast_api_image.add_local_dir(local_assets_path, remote_path="/assets") @app.function( - image=modal.Image.debian_slim().pip_install("fastapi[standard]==0.115.4").add_local_dir(assets_path, remote_path="/assets"), + image=ocr_app_image ) @modal.asgi_app() def wrapper(): web_app.mount( "/", fastapi.staticfiles.StaticFiles(directory="/assets", html=True) ) - return web_app diff --git a/10_integrations/dbt_modal_inference/dbt_modal_inference.py b/10_integrations/dbt_modal_inference/dbt_modal_inference.py index d242756bd..e776d34f0 100644 --- a/10_integrations/dbt_modal_inference/dbt_modal_inference.py +++ b/10_integrations/dbt_modal_inference/dbt_modal_inference.py @@ -51,13 +51,14 @@ "DB_PATH": DB_PATH, } ) + # We mount the local code and configuration into the Modal Function + # so that it will be available when we run dbt + # and create a volume so that we can persist our data. ) app = modal.App("duckdb-dbt-inference", image=dbt_image) -# We mount the local code and configuration into the Modal Function -# so that it will be available when we run dbt -# and create a volume so that we can persist our data. + dbt_project = modal.Mount.from_local_dir( LOCAL_DBT_PROJECT, remote_path=PROJ_PATH From 94ffb5cab90698f7787ec9a6c52b600b180c9d36 Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Tue, 26 Nov 2024 13:18:13 +0100 Subject: [PATCH 07/20] Remove mount from text generation inference --- 06_gpu_and_ml/llm-serving/text_generation_inference.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/06_gpu_and_ml/llm-serving/text_generation_inference.py b/06_gpu_and_ml/llm-serving/text_generation_inference.py index 79a85d18b..6f5492063 100644 --- a/06_gpu_and_ml/llm-serving/text_generation_inference.py +++ b/06_gpu_and_ml/llm-serving/text_generation_inference.py @@ -203,9 +203,15 @@ def main(prompt: str = None): frontend_path = Path(__file__).parent.parent / "llm-frontend" +# Create an image with fastapi and the local web assets added to /assets +webapp_image = ( + modal.Image.debian_slim() + .pip_install("fastapi") + .add_local_dir(frontend_path, remote_path="/assets") +) @app.function( - mounts=[modal.Mount.from_local_dir(frontend_path, remote_path="/assets")], + image=webapp_image, keep_warm=1, allow_concurrent_inputs=10, timeout=60 * 10, From cf702e6bafcd90eb3be8d4cfb0df5956d337d4f4 Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Tue, 26 Nov 2024 13:19:19 +0100 Subject: [PATCH 08/20] tgi mixtral --- 06_gpu_and_ml/llm-serving/text_generation_inference.py | 2 +- 06_gpu_and_ml/llm-serving/tgi_mixtral.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/06_gpu_and_ml/llm-serving/text_generation_inference.py b/06_gpu_and_ml/llm-serving/text_generation_inference.py index 6f5492063..13cbfbf09 100644 --- a/06_gpu_and_ml/llm-serving/text_generation_inference.py +++ b/06_gpu_and_ml/llm-serving/text_generation_inference.py @@ -206,7 +206,7 @@ def main(prompt: str = None): # Create an image with fastapi and the local web assets added to /assets webapp_image = ( modal.Image.debian_slim() - .pip_install("fastapi") + .pip_install("fastapi[standard]") .add_local_dir(frontend_path, remote_path="/assets") ) diff --git a/06_gpu_and_ml/llm-serving/tgi_mixtral.py b/06_gpu_and_ml/llm-serving/tgi_mixtral.py index 1efbd9b1c..1626b7a68 100644 --- a/06_gpu_and_ml/llm-serving/tgi_mixtral.py +++ b/06_gpu_and_ml/llm-serving/tgi_mixtral.py @@ -182,12 +182,13 @@ def main(): # You can try our deployment [here](https://modal-labs--tgi-mixtral.modal.run). frontend_path = Path(__file__).parent.parent / "llm-frontend" -frontend_image = modal.Image.debian_slim().pip_install("fastapi[standard]") - +frontend_image = ( + modal.Image.debian_slim().pip_install("fastapi[standard]") + .add_local_dir(frontend_path, remote_path="/assets") +) @app.function( image=frontend_image, - mounts=[modal.Mount.from_local_dir(frontend_path, remote_path="/assets")], keep_warm=1, allow_concurrent_inputs=20, timeout=60 * 10, From 8232405f8245af40947190f6068ff68446bee23a Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Tue, 26 Nov 2024 13:38:11 +0100 Subject: [PATCH 09/20] dbt modal inference --- .../dbt_modal_inference.py | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/10_integrations/dbt_modal_inference/dbt_modal_inference.py b/10_integrations/dbt_modal_inference/dbt_modal_inference.py index e776d34f0..38350f2e9 100644 --- a/10_integrations/dbt_modal_inference/dbt_modal_inference.py +++ b/10_integrations/dbt_modal_inference/dbt_modal_inference.py @@ -51,22 +51,21 @@ "DB_PATH": DB_PATH, } ) - # We mount the local code and configuration into the Modal Function + # We add the local code and configuration into the image # so that it will be available when we run dbt - # and create a volume so that we can persist our data. + .add_local_dir( + LOCAL_DBT_PROJECT, remote_path=PROJ_PATH + ) + .add_local_file( + local_path=LOCAL_DBT_PROJECT / "profiles.yml", + remote_path=f"{PROFILES_PATH}/profiles.yml", + ) ) app = modal.App("duckdb-dbt-inference", image=dbt_image) - -dbt_project = modal.Mount.from_local_dir( - LOCAL_DBT_PROJECT, remote_path=PROJ_PATH -) -dbt_profiles = modal.Mount.from_local_file( - local_path=LOCAL_DBT_PROJECT / "profiles.yml", - remote_path=pathlib.Path(PROFILES_PATH, "profiles.yml"), -) +# Create a modal.Volume so that we can persist our data dbt_vol = modal.Volume.from_name("dbt-inference-vol", create_if_missing=True) # ## Run dbt in a serverless Modal Function @@ -85,7 +84,6 @@ @app.function( - mounts=[dbt_project, dbt_profiles], volumes={VOL_PATH: dbt_vol}, ) def dbt_run() -> None: @@ -97,8 +95,8 @@ def dbt_run() -> None: os.makedirs(DB_PATH, exist_ok=True) os.makedirs(TARGET_PATH, exist_ok=True) - # Remember to either deploy this yourself in your environment - # or change to another web endpoint you have + # Remember to either deploy the llama dependency app in your environment + # first, or change this to use another web endpoint you have: ref = modal.Function.lookup( "example-trtllm-Meta-Llama-3-8B-Instruct", "generate_web" ) From 8696577f8ee314d52c877f240d5fe27561c5401c Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Tue, 26 Nov 2024 13:41:19 +0100 Subject: [PATCH 10/20] Dbt duckdb example --- 10_integrations/dbt/dbt_duckdb.py | 39 ++++++++++++++----------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/10_integrations/dbt/dbt_duckdb.py b/10_integrations/dbt/dbt_duckdb.py index 2144b687b..459e53b37 100644 --- a/10_integrations/dbt/dbt_duckdb.py +++ b/10_integrations/dbt/dbt_duckdb.py @@ -34,6 +34,15 @@ PROFILES_PATH = "/root/dbt_profile" TARGET_PATH = "/root/target" +# Most of the DBT code and configuration is taken directly from the classic +# [Jaffle Shop](https://github.com/dbt-labs/jaffle_shop) demo and modified to support +# using `dbt-duckdb` with an S3 bucket. +# +# The DBT `profiles.yml` configuration is taken from +# [the `dbt-duckdb` docs](https://github.com/jwills/dbt-duckdb#configuring-your-profile). + + + # We also define the environment our application will run in -- # a container image, as in Docker. # See [this guide](https://modal.com/docs/guide/custom-container) for details. @@ -54,27 +63,18 @@ "DBT_TARGET_PATH": TARGET_PATH, } ) + # Here we add all local code and configuration into the Modal Image + # so that it will be available when we run DBT on Modal. + .add_local_dir( + LOCAL_DBT_PROJECT, remote_path=PROJ_PATH + ) + .add_local_file( + local_path=LOCAL_DBT_PROJECT / "profiles.yml", + remote_path=f"{PROFILES_PATH}/profiles.yml", + ) ) app = modal.App(name="example-dbt-duckdb-s3", image=dbt_image) - -# Most of the DBT code and configuration is taken directly from the classic -# [Jaffle Shop](https://github.com/dbt-labs/jaffle_shop) demo and modified to support -# using `dbt-duckdb` with an S3 bucket. -# -# The DBT `profiles.yml` configuration is taken from -# [the `dbt-duckdb` docs](https://github.com/jwills/dbt-duckdb#configuring-your-profile). -# -# Here we mount all this local code and configuration into the Modal Function -# so that it will be available when we run DBT on Modal. - -dbt_project = modal.Mount.from_local_dir( - LOCAL_DBT_PROJECT, remote_path=PROJ_PATH -) -dbt_profiles = modal.Mount.from_local_file( - local_path=LOCAL_DBT_PROJECT / "profiles.yml", - remote_path=Path(PROFILES_PATH, "profiles.yml"), -) dbt_target = modal.Volume.from_name("dbt-target-vol", create_if_missing=True) # We'll also need to authenticate with AWS to store data in S3. @@ -128,7 +128,6 @@ @app.function( - mounts=[dbt_project], secrets=[s3_secret], ) def create_source_data(): @@ -171,7 +170,6 @@ def create_source_data(): @app.function( secrets=[s3_secret], - mounts=[dbt_project, dbt_profiles], volumes={TARGET_PATH: dbt_target}, ) def run(command: str) -> None: @@ -273,7 +271,6 @@ def serve_dbt_docs(): @app.function( schedule=modal.Period(days=1), secrets=[s3_secret], - mounts=[dbt_project, dbt_profiles], volumes={TARGET_PATH: dbt_target}, ) def daily_build() -> None: From 7de2eee3ca9ede05edeca91543f5cf13b3e85fe4 Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Tue, 26 Nov 2024 13:46:03 +0100 Subject: [PATCH 11/20] fasthtml checkboxes --- .../fasthtml-checkboxes/fasthtml_checkboxes.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/07_web_endpoints/fasthtml-checkboxes/fasthtml_checkboxes.py b/07_web_endpoints/fasthtml-checkboxes/fasthtml_checkboxes.py index f3d2b773b..ed1d8c8cb 100644 --- a/07_web_endpoints/fasthtml-checkboxes/fasthtml_checkboxes.py +++ b/07_web_endpoints/fasthtml-checkboxes/fasthtml_checkboxes.py @@ -19,7 +19,7 @@ import time from asyncio import Lock -from pathlib import Path +from pathlib import Path, PosixPath from uuid import uuid4 import modal @@ -30,17 +30,14 @@ db = modal.Dict.from_name("example-checkboxes-db", create_if_missing=True) css_path_local = Path(__file__).parent / "styles.css" -css_path_remote = Path("/assets/styles.css") +css_path_remote = "/assets/styles.css" @app.function( image=modal.Image.debian_slim(python_version="3.12").pip_install( "python-fasthtml==0.6.9", "inflect~=7.4.0" - ), + ).add_local_file(css_path_local, remote_path=css_path_remote), concurrency_limit=1, # we currently maintain state in memory, so we restrict the server to one worker - mounts=[ - modal.Mount.from_local_file(css_path_local, remote_path=css_path_remote) - ], allow_concurrent_inputs=1000, ) @modal.asgi_app() From 94bcfcfb375473ab661070232ffbe9d87f290464 Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Tue, 26 Nov 2024 13:54:52 +0100 Subject: [PATCH 12/20] hp sweep gpt --- .../hyperparameter-sweep/hp_sweep_gpt.py | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/06_gpu_and_ml/hyperparameter-sweep/hp_sweep_gpt.py b/06_gpu_and_ml/hyperparameter-sweep/hp_sweep_gpt.py index 80a34c1f2..017bf916d 100644 --- a/06_gpu_and_ml/hyperparameter-sweep/hp_sweep_gpt.py +++ b/06_gpu_and_ml/hyperparameter-sweep/hp_sweep_gpt.py @@ -50,7 +50,7 @@ import logging as L import urllib.request from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PosixPath import modal from pydantic import BaseModel @@ -75,7 +75,7 @@ volume = modal.Volume.from_name( "example-hp-sweep-gpt-volume", create_if_missing=True ) -volume_path = Path("/vol/data") +volume_path = PosixPath("/vol/data") model_filename = "nano_gpt_model.pt" best_model_filename = "best_nano_gpt_model.pt" tb_log_path = volume_path / "tb_logs" @@ -93,24 +93,22 @@ "torch==2.1.2", "tensorboard==2.17.1", "numpy<2", -) +).add_local_dir(Path(__file__).parent / "src", remote_path="/root/src") # We also have some local dependencies that we'll need to import into the remote environment. # We mount them onto the remote container. -mounts = [ - modal.Mount.from_local_dir( - Path(__file__).parent / "src", remote_path=Path("/root/src") - ) -] - # We'll serve a simple web endpoint web_image = base_image.pip_install( "fastapi[standard]==0.115.4", "starlette==0.41.2" ) # And we'll deploy a web UI for interacting with our trained models using Gradio. -ui_image = web_image.pip_install("gradio~=4.44.0") +assets_path = Path(__file__).parent / "assets" +ui_image = web_image.pip_install("gradio~=4.44.0").add_local_dir( + assets_path, remote_path="/assets" +) + # We can also "pre-import" libraries that will be used by the functions we run on Modal in a given image # using the `with image.imports` context manager. @@ -140,7 +138,6 @@ @app.function( image=torch_image, - mounts=mounts, volumes={volume_path: volume}, gpu=gpu, timeout=1 * HOURS, @@ -576,7 +573,6 @@ def web_generate(request: GenerationRequest): # The Gradio UI will look something like this: # ![Image of Gradio Web App. Top shows model selection dropdown. Left side shows input prompt textbox. Right side shows SLM generated output. Bottom has button for starting generation process](./gradio.png) -assets_path = Path(__file__).parent / "assets" @app.function( @@ -584,7 +580,6 @@ def web_generate(request: GenerationRequest): concurrency_limit=1, volumes={volume_path: volume}, allow_concurrent_inputs=1000, - mounts=[modal.Mount.from_local_dir(assets_path, remote_path="/assets")], ) @modal.asgi_app() def ui(): From e3a7eb7133ea0846a801ec151863bd22d0d3e8bd Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Tue, 26 Nov 2024 13:57:17 +0100 Subject: [PATCH 13/20] more --- 06_gpu_and_ml/obj_detection_webcam/webcam.py | 5 +++-- 06_gpu_and_ml/stable_diffusion/text_to_image.py | 7 ++++--- 06_gpu_and_ml/text-to-pokemon/text_to_pokemon/config.py | 1 + 06_gpu_and_ml/text-to-pokemon/text_to_pokemon/main.py | 8 +------- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/06_gpu_and_ml/obj_detection_webcam/webcam.py b/06_gpu_and_ml/obj_detection_webcam/webcam.py index e94c3f4ed..e56851faf 100644 --- a/06_gpu_and_ml/obj_detection_webcam/webcam.py +++ b/06_gpu_and_ml/obj_detection_webcam/webcam.py @@ -167,8 +167,9 @@ def detect(self, img_data_in): @app.function( - image=modal.Image.debian_slim().pip_install("fastapi[standard]"), - mounts=[modal.Mount.from_local_dir(static_path, remote_path="/assets")], + image=modal.Image.debian_slim() + .pip_install("fastapi[standard]") + .add_local_dir(static_path, remote_path="/assets") ) @modal.asgi_app() def fastapi_app(): diff --git a/06_gpu_and_ml/stable_diffusion/text_to_image.py b/06_gpu_and_ml/stable_diffusion/text_to_image.py index 56c906b1e..a5356f095 100644 --- a/06_gpu_and_ml/stable_diffusion/text_to_image.py +++ b/06_gpu_and_ml/stable_diffusion/text_to_image.py @@ -223,14 +223,15 @@ def entrypoint( frontend_path = Path(__file__).parent / "frontend" -web_image = modal.Image.debian_slim(python_version="3.12").pip_install( - "jinja2==3.1.4", "fastapi[standard]==0.115.4" +web_image = ( + modal.Image.debian_slim(python_version="3.12") + .pip_install("jinja2==3.1.4", "fastapi[standard]==0.115.4") + .add_local_dir(frontend_path, remote_path="/assets") ) @app.function( image=web_image, - mounts=[modal.Mount.from_local_dir(frontend_path, remote_path="/assets")], allow_concurrent_inputs=1000, ) @modal.asgi_app() diff --git a/06_gpu_and_ml/text-to-pokemon/text_to_pokemon/config.py b/06_gpu_and_ml/text-to-pokemon/text_to_pokemon/config.py index fd966c818..5990dff7d 100644 --- a/06_gpu_and_ml/text-to-pokemon/text_to_pokemon/config.py +++ b/06_gpu_and_ml/text-to-pokemon/text_to_pokemon/config.py @@ -206,5 +206,6 @@ def null_safety(images, **kwargs): "fastapi[standard]", ) .run_function(load_stable_diffusion_pokemon_model) + .add_local_dir(local_path=ASSETS_PATH, remote_path="/assets") ) app = modal.App(name="example-text-to-pokemon", image=image) diff --git a/06_gpu_and_ml/text-to-pokemon/text_to_pokemon/main.py b/06_gpu_and_ml/text-to-pokemon/text_to_pokemon/main.py index cae9d2db1..cee4bf682 100644 --- a/06_gpu_and_ml/text-to-pokemon/text_to_pokemon/main.py +++ b/06_gpu_and_ml/text-to-pokemon/text_to_pokemon/main.py @@ -129,13 +129,7 @@ def diskcached_text_to_pokemon(prompt: str) -> list[bytes]: return samples_data -@app.function( - mounts=[ - modal.Mount.from_local_dir( - local_path=config.ASSETS_PATH, remote_path="/assets" - ) - ], -) +@app.function() @modal.asgi_app() def fastapi_app(): import fastapi.staticfiles From 36eaa7e86fd8cdb0c4ea11eae2b60fe49ab412e7 Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Tue, 26 Nov 2024 14:21:33 +0100 Subject: [PATCH 14/20] Podcast --- 06_gpu_and_ml/openai_whisper/pod_transcriber/app/main.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/06_gpu_and_ml/openai_whisper/pod_transcriber/app/main.py b/06_gpu_and_ml/openai_whisper/pod_transcriber/app/main.py index bed69baeb..6c0a3ff64 100644 --- a/06_gpu_and_ml/openai_whisper/pod_transcriber/app/main.py +++ b/06_gpu_and_ml/openai_whisper/pod_transcriber/app/main.py @@ -93,9 +93,7 @@ def populate_podcast_metadata(podcast_id: str): @app.function( - mounts=[ - modal.Mount.from_local_dir(config.ASSETS_PATH, remote_path="/assets") - ], + image=app_image.add_local_dir(config.ASSETS_PATH, remote_path="/assets"), network_file_systems={config.CACHE_DIR: volume}, keep_warm=2, ) @@ -112,9 +110,7 @@ def fastapi_app(): return web_app -@app.function( - image=app_image, -) +@app.function() def search_podcast(name): from gql import gql From 9080b8949f7d0d5df0828a212a5794d0f130ceb0 Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Tue, 26 Nov 2024 14:29:55 +0100 Subject: [PATCH 15/20] Updated some text --- 06_gpu_and_ml/comfyui/comfyapp.py | 2 +- 10_integrations/dbt/dbt_duckdb.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/06_gpu_and_ml/comfyui/comfyapp.py b/06_gpu_and_ml/comfyui/comfyapp.py index 0d3665e1f..3a862ee29 100644 --- a/06_gpu_and_ml/comfyui/comfyapp.py +++ b/06_gpu_and_ml/comfyui/comfyapp.py @@ -91,7 +91,7 @@ # ### Downloading models # You can also use comfy-cli to download models, but for this example we'll download the Flux models directly from Hugging Face into a Modal Volume. -# Then on container start, we'll mount our models into the ComfyUI models directory. +# Then on container start, we'll mount our models Volume into the ComfyUI models directory. # This allows us to avoid re-downloading the models every time you rebuild your image. image = ( diff --git a/10_integrations/dbt/dbt_duckdb.py b/10_integrations/dbt/dbt_duckdb.py index 459e53b37..35c6cb8b7 100644 --- a/10_integrations/dbt/dbt_duckdb.py +++ b/10_integrations/dbt/dbt_duckdb.py @@ -42,7 +42,6 @@ # [the `dbt-duckdb` docs](https://github.com/jwills/dbt-duckdb#configuring-your-profile). - # We also define the environment our application will run in -- # a container image, as in Docker. # See [this guide](https://modal.com/docs/guide/custom-container) for details. @@ -65,9 +64,7 @@ ) # Here we add all local code and configuration into the Modal Image # so that it will be available when we run DBT on Modal. - .add_local_dir( - LOCAL_DBT_PROJECT, remote_path=PROJ_PATH - ) + .add_local_dir(LOCAL_DBT_PROJECT, remote_path=PROJ_PATH) .add_local_file( local_path=LOCAL_DBT_PROJECT / "profiles.yml", remote_path=f"{PROFILES_PATH}/profiles.yml", @@ -163,8 +160,8 @@ def create_source_data(): # below, we import the `dbt` library's `dbtRunner` to pass commands from our # Python code, running on Modal, the same way we'd pass commands on a command line. # -# Note that this Modal Function has access to our AWS Secret, -# the `mount`ed local files with our DBT project and profiles, +# Note that this Modal Function has access to our AWS S3 Secret, +# the local files associated with our DBT project and profiles, # and a remote Modal Volume that acts as a distributed file system. From ae3e3365f686074d3f589aa1b9da0db3769fdaca Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Tue, 26 Nov 2024 14:33:33 +0100 Subject: [PATCH 16/20] Ruff --- .../fasthtml-checkboxes/fasthtml_checkboxes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/07_web_endpoints/fasthtml-checkboxes/fasthtml_checkboxes.py b/07_web_endpoints/fasthtml-checkboxes/fasthtml_checkboxes.py index ed1d8c8cb..89ca4d27e 100644 --- a/07_web_endpoints/fasthtml-checkboxes/fasthtml_checkboxes.py +++ b/07_web_endpoints/fasthtml-checkboxes/fasthtml_checkboxes.py @@ -19,7 +19,7 @@ import time from asyncio import Lock -from pathlib import Path, PosixPath +from pathlib import Path from uuid import uuid4 import modal @@ -34,9 +34,9 @@ @app.function( - image=modal.Image.debian_slim(python_version="3.12").pip_install( - "python-fasthtml==0.6.9", "inflect~=7.4.0" - ).add_local_file(css_path_local, remote_path=css_path_remote), + image=modal.Image.debian_slim(python_version="3.12") + .pip_install("python-fasthtml==0.6.9", "inflect~=7.4.0") + .add_local_file(css_path_local, remote_path=css_path_remote), concurrency_limit=1, # we currently maintain state in memory, so we restrict the server to one worker allow_concurrent_inputs=1000, ) From 7f1b8b8c448687d5b1528f94847b73f039e40967 Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Tue, 26 Nov 2024 14:40:36 +0100 Subject: [PATCH 17/20] Ruff format --- .../dreambooth/diffusers_lora_finetune.py | 5 +++-- .../llm-serving/text_generation_inference.py | 5 +++-- 06_gpu_and_ml/llm-serving/tgi_mixtral.py | 6 ++++-- 09_job_queues/doc_ocr_webapp.py | 14 +++++++++----- .../dbt_modal_inference/dbt_modal_inference.py | 4 +--- 10_integrations/streamlit/serve_streamlit.py | 12 +++++++----- 6 files changed, 27 insertions(+), 19 deletions(-) diff --git a/06_gpu_and_ml/dreambooth/diffusers_lora_finetune.py b/06_gpu_and_ml/dreambooth/diffusers_lora_finetune.py index b1785f41a..2e560ef35 100644 --- a/06_gpu_and_ml/dreambooth/diffusers_lora_finetune.py +++ b/06_gpu_and_ml/dreambooth/diffusers_lora_finetune.py @@ -141,6 +141,7 @@ class SharedConfig: {"HF_HUB_ENABLE_HF_TRANSFER": "1"} # turn on faster downloads from HF ) + @app.function( volumes={MODEL_DIR: volume}, image=image, @@ -417,10 +418,10 @@ class AppConfig(SharedConfig): # attach local web assets image_with_assets = image.add_local_dir( - Path(__file__).parent / "assets", - remote_path="/assets" + Path(__file__).parent / "assets", remote_path="/assets" ) + @app.function( image=image_with_assets, concurrency_limit=1, diff --git a/06_gpu_and_ml/llm-serving/text_generation_inference.py b/06_gpu_and_ml/llm-serving/text_generation_inference.py index 13cbfbf09..302a0c511 100644 --- a/06_gpu_and_ml/llm-serving/text_generation_inference.py +++ b/06_gpu_and_ml/llm-serving/text_generation_inference.py @@ -206,10 +206,11 @@ def main(prompt: str = None): # Create an image with fastapi and the local web assets added to /assets webapp_image = ( modal.Image.debian_slim() - .pip_install("fastapi[standard]") - .add_local_dir(frontend_path, remote_path="/assets") + .pip_install("fastapi[standard]") + .add_local_dir(frontend_path, remote_path="/assets") ) + @app.function( image=webapp_image, keep_warm=1, diff --git a/06_gpu_and_ml/llm-serving/tgi_mixtral.py b/06_gpu_and_ml/llm-serving/tgi_mixtral.py index 1626b7a68..600582323 100644 --- a/06_gpu_and_ml/llm-serving/tgi_mixtral.py +++ b/06_gpu_and_ml/llm-serving/tgi_mixtral.py @@ -183,10 +183,12 @@ def main(): frontend_path = Path(__file__).parent.parent / "llm-frontend" frontend_image = ( - modal.Image.debian_slim().pip_install("fastapi[standard]") - .add_local_dir(frontend_path, remote_path="/assets") + modal.Image.debian_slim() + .pip_install("fastapi[standard]") + .add_local_dir(frontend_path, remote_path="/assets") ) + @app.function( image=frontend_image, keep_warm=1, diff --git a/09_job_queues/doc_ocr_webapp.py b/09_job_queues/doc_ocr_webapp.py index 152f658c4..28b2e3abe 100644 --- a/09_job_queues/doc_ocr_webapp.py +++ b/09_job_queues/doc_ocr_webapp.py @@ -74,9 +74,12 @@ async def poll_results(call_id: str): return result + # Specify a container image containing the version of fastapi we want to use to serve # our web app -fast_api_image = modal.Image.debian_slim().pip_install("fastapi[standard]==0.115.4") +fast_api_image = modal.Image.debian_slim().pip_install( + "fastapi[standard]==0.115.4" +) # Finally, we add the static files for our front-end. We've made [a simple React # app](https://github.com/modal-labs/modal-examples/tree/main/09_job_queues/doc_ocr_frontend) @@ -86,11 +89,12 @@ async def poll_results(call_id: str): # this static file directory](https://fastapi.tiangolo.com/tutorial/static-files/) at our root path. local_assets_path = Path(__file__).parent / "doc_ocr_frontend" -ocr_app_image = fast_api_image.add_local_dir(local_assets_path, remote_path="/assets") - -@app.function( - image=ocr_app_image +ocr_app_image = fast_api_image.add_local_dir( + local_assets_path, remote_path="/assets" ) + + +@app.function(image=ocr_app_image) @modal.asgi_app() def wrapper(): web_app.mount( diff --git a/10_integrations/dbt_modal_inference/dbt_modal_inference.py b/10_integrations/dbt_modal_inference/dbt_modal_inference.py index 38350f2e9..e1c634252 100644 --- a/10_integrations/dbt_modal_inference/dbt_modal_inference.py +++ b/10_integrations/dbt_modal_inference/dbt_modal_inference.py @@ -53,9 +53,7 @@ ) # We add the local code and configuration into the image # so that it will be available when we run dbt - .add_local_dir( - LOCAL_DBT_PROJECT, remote_path=PROJ_PATH - ) + .add_local_dir(LOCAL_DBT_PROJECT, remote_path=PROJ_PATH) .add_local_file( local_path=LOCAL_DBT_PROJECT / "profiles.yml", remote_path=f"{PROFILES_PATH}/profiles.yml", diff --git a/10_integrations/streamlit/serve_streamlit.py b/10_integrations/streamlit/serve_streamlit.py index 38bc6331b..47079306a 100644 --- a/10_integrations/streamlit/serve_streamlit.py +++ b/10_integrations/streamlit/serve_streamlit.py @@ -29,11 +29,13 @@ streamlit_script_local_path = Path(__file__).parent / "app.py" streamlit_script_remote_path = Path("/root/app.py") -image = modal.Image.debian_slim(python_version="3.11").pip_install( - "streamlit~=1.35.0", "numpy~=1.26.4", "pandas~=2.2.2" -).add_local_file( - streamlit_script_local_path, - streamlit_script_remote_path, +image = ( + modal.Image.debian_slim(python_version="3.11") + .pip_install("streamlit~=1.35.0", "numpy~=1.26.4", "pandas~=2.2.2") + .add_local_file( + streamlit_script_local_path, + streamlit_script_remote_path, + ) ) app = modal.App(name="example-modal-streamlit", image=image) From f0b45f2edd9418349bb7bbbc16bf69e85fec1a58 Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Tue, 26 Nov 2024 15:07:36 +0100 Subject: [PATCH 18/20] reformat --- 06_gpu_and_ml/dreambooth/diffusers_lora_finetune.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/06_gpu_and_ml/dreambooth/diffusers_lora_finetune.py b/06_gpu_and_ml/dreambooth/diffusers_lora_finetune.py index 2e560ef35..1fc492e1c 100644 --- a/06_gpu_and_ml/dreambooth/diffusers_lora_finetune.py +++ b/06_gpu_and_ml/dreambooth/diffusers_lora_finetune.py @@ -416,9 +416,10 @@ class AppConfig(SharedConfig): guidance_scale: float = 6 -# attach local web assets image_with_assets = image.add_local_dir( - Path(__file__).parent / "assets", remote_path="/assets" + # Add local web assets to the image + Path(__file__).parent / "assets", + remote_path="/assets", ) From f330c19e4a149640f7eb24a558de09b4c900ae0c Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Fri, 13 Dec 2024 13:41:07 +0100 Subject: [PATCH 19/20] Remove kedro example that hasn't been updated in a long time Uses a bunch of deprecated defunct functionality but nobody has complained, so we can probably remove and add it back if someone asks for it --- 10_integrations/kedro_modal/README.md | 70 -- .../kedro_modal/kedro_modal/cli.py | 66 -- .../kedro_modal/modal_functions.py | 94 -- 10_integrations/kedro_modal/poetry.lock | 1054 ----------------- 10_integrations/kedro_modal/pyproject.toml | 22 - 5 files changed, 1306 deletions(-) delete mode 100644 10_integrations/kedro_modal/README.md delete mode 100644 10_integrations/kedro_modal/kedro_modal/cli.py delete mode 100644 10_integrations/kedro_modal/kedro_modal/modal_functions.py delete mode 100644 10_integrations/kedro_modal/poetry.lock delete mode 100644 10_integrations/kedro_modal/pyproject.toml diff --git a/10_integrations/kedro_modal/README.md b/10_integrations/kedro_modal/README.md deleted file mode 100644 index b4612afc7..000000000 --- a/10_integrations/kedro_modal/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Kedro-Modal (A plugin for Kedro) - -kedro-modal is an experimental Modal plugin for Kedro (https://kedro.readthedocs.io/en/stable/). - -The plugin lets you run Kedro Python pipelines on Modal in an effortless way. - -Data in your `/data` directory will be synced to a persisted Modal [Network File System](/docs/guide/network-file-systems) and any datasets defined as local in your Kedro data catalog will be written to the same volume. - -## Installation instructions - -First make sure that you have Modal installed (See https://modal.com/home) - -```bash -cd modal-examples/integrations/kedro-modal -pip install . -``` - -This installs the kedro-modal project cli commands, e.g. `kedro modal --help` - -## Usage - -The following commands assume you have navigated to your kedro project, e.g.: -`cd my_kedro_project` - -### Run a Kedro pipeline on Modal: - -```bash -kedro modal run -``` - -### Reset the remote data volume using local data - -```bash -kedro modal reset -``` - -## Inspect and download output data - -### List Kedro data volumes on Modal: - -```bash -modal volume list | grep '^kedro.*storage$' -``` - -Example output: - -``` -kedro.Spaceflights.storage -``` - -### Download an output file (example): - -```bash -modal volume get kedro.Spaceflights.storage data/02_intermediate/preprocessed_shuttles.pq . -``` - -## Features - -At this point it only supports the basic use case of running a Kedro project (equivlalent of a basic `kedro run`) on Modal instead of running it locally. - -- It pushes the project source code to Modal -- Installs requirements.txt of the project in a Modal image -- Sets up a modal Network File System for syncing local input data from project/data and writing any output from the Modal runs. - -### Notably missing features at this point (but should not be too tricky to handle): - -- External data processors (e.g. spark) -- Parallel runs using a custom Kedro Runner subclass -- Default dataset support using Modal distributed in-memory storage (i.e. modal.Dict) - mostly needed if running parallelism -- Using Modal functions as explicit Kedro nodes/steps (should technically be doable by just using Modal as is, but would probably need some kind of integration with this plugin) diff --git a/10_integrations/kedro_modal/kedro_modal/cli.py b/10_integrations/kedro_modal/kedro_modal/cli.py deleted file mode 100644 index 2d986a594..000000000 --- a/10_integrations/kedro_modal/kedro_modal/cli.py +++ /dev/null @@ -1,66 +0,0 @@ -import click -import modal - -from .modal_functions import main_app, sync_app - - -@click.group(name="Kedro-Modal") -def commands(): - """Kedro plugin for running kedro pipelines on Modal""" - pass - - -@commands.group( - name="modal", context_settings=dict(help_option_names=["-h", "--help"]) -) -def modal_group(): - """Interact with Kedro pipelines run on Modal""" - - -@modal_group.command(help="Run kedro project on Modal") -@click.pass_obj -def run(metadata): - app, remote_project_mount_path, remote_data_path = main_app( - metadata.project_path, metadata.project_name, metadata.package_name - ) - with app.run() as app: - app.sync_data( - remote_project_mount_path / "data", remote_data_path, reset=False - ) - app.run_kedro(remote_project_mount_path, remote_data_path) - - -@modal_group.command(help="Run kedro project on Modal") -@click.pass_obj -def debug(metadata): - app, remote_project_mount_path, remote_data_path = main_app( - metadata.project_path, metadata.project_name, metadata.package_name - ) - app.interactive_shell() - - -@modal_group.command( - help="Deploy kedro project to Modal, scheduling it to run daily" -) -@click.pass_obj -def deploy(metadata): - app, remote_project_mount_point, remote_data_path = main_app( - metadata.project_path, metadata.project_name, metadata.package_name - ) - name = f"kedro.{metadata.project_name}" - app.deploy(name) - sync_data = modal.lookup(name, "sync_data") # use the deployed function - sync_data(remote_project_mount_point / "data", remote_data_path) - - -@modal_group.command( - short_help="Sync the local data directory to Modal", - help="Sync the local data directory to Modal, overwriting any existing data there", -) -@click.pass_obj -def reset(metadata): - app, source_path, destination_path = sync_app( - metadata.project_path, metadata.project_name - ) - with app.run() as app: - app.sync_data(source_path, destination_path, reset=True) diff --git a/10_integrations/kedro_modal/kedro_modal/modal_functions.py b/10_integrations/kedro_modal/kedro_modal/modal_functions.py deleted file mode 100644 index d5329c1a7..000000000 --- a/10_integrations/kedro_modal/kedro_modal/modal_functions.py +++ /dev/null @@ -1,94 +0,0 @@ -import os -import shutil -import subprocess -import warnings -from pathlib import Path - -import modal - -package_mounts = modal.create_package_mounts(["kedro_modal"]) - - -def run_kedro(project_path: Path, data_path: Path): - shutil.rmtree(project_path / "data") # replace project mounted data dir - (project_path / "data").symlink_to(data_path) - os.chdir(project_path) - subprocess.call(["kedro", "run"]) - - -def sync_data(source: Path, destination: Path, reset: bool = False): - """Sync a local data directory *to* a network file system""" - - # TODO: only sync raw data - no intermediates etc? - if destination.exists() and reset: - shutil.rmtree(destination) - if not destination.exists(): - shutil.copytree(source, destination) - - -def non_hidden_files(project_path: Path): - def condition(path): - rel = Path(path).relative_to(project_path) - return not any( - part != ".gitkeep" and part.startswith(".") for part in rel.parts - ) - - return condition - - -def main_app(project_path, project_name, package_name) -> modal.App: - requirements_txt = project_path / "src" / "requirements.txt" - - image = modal.Image.debian_slim() - if requirements_txt.exists(): - image = image.pip_install_from_requirements(requirements_txt) - else: - warnings.warn( - "No requirements.txt in kedro src dir - attaching no dependencies" - ) - image = image.pip_install("kedro") - - remote_project_mount_point = Path(f"/kedro-project/{package_name}") - kedro_proj_mount = modal.Mount( - remote_dir=remote_project_mount_point, - local_dir=project_path, - condition=non_hidden_files(project_path), - ) - app = modal.App( - f"kedro-run.{project_name}", - image=image, - mounts=[kedro_proj_mount] + package_mounts, - ) - volume_name = f"kedro.{project_name}.storage" - data_volume = modal.NetworkFileSystem.from_name( - volume_name, create_if_true=True - ) - - app.function(network_file_systems={"/kedro-storage": data_volume})( - run_kedro - ) - app.function(network_file_systems={"/kedro-storage": data_volume})( - sync_data - ) - remote_data_path = Path("/kedro-storage/data") - return app, remote_project_mount_point, remote_data_path - - -def sync_app(project_path, project_name): - # slimmer sync app that only mounts the data dir in order to upload raw data - app = modal.App(f"kedro-data-sync.{project_name}") - volume_name = f"kedro.{project_name}.storage" - data_volume = modal.NetworkFileSystem().persist(volume_name) - - remote_source_path = Path("/source-data") - source_mount = modal.Mount( - remote_dir=remote_source_path, - local_dir=project_path / "data", - condition=non_hidden_files(project_path), - ) - app.function( - mounts=[source_mount] + package_mounts, - network_file_systems={"/kedro-storage": data_volume}, - )(sync_data) - remote_destination_path = Path("/kedro-storage/data") - return app, remote_source_path, remote_destination_path diff --git a/10_integrations/kedro_modal/poetry.lock b/10_integrations/kedro_modal/poetry.lock deleted file mode 100644 index be4d09b62..000000000 --- a/10_integrations/kedro_modal/poetry.lock +++ /dev/null @@ -1,1054 +0,0 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. - -[[package]] -name = "antlr4-python3-runtime" -version = "4.9.3" -description = "ANTLR 4.9.3 runtime for Python 3.7" -optional = false -python-versions = "*" -files = [ - {file = "antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b"}, -] - -[[package]] -name = "anyconfig" -version = "0.10.1" -description = "Library provides common APIs to load and dump configuration files in various formats" -optional = false -python-versions = "*" -files = [ - {file = "anyconfig-0.10.1-py2.py3-none-any.whl", hash = "sha256:60c795cc6ef3e6f0a74182c52dfa6e8825d852b2b9128404e1aef3b84ca077a4"}, - {file = "anyconfig-0.10.1.tar.gz", hash = "sha256:f04a5490da8563c97fad15810b0debc92351dbd4b8058dfd82d32a30a41e7e5c"}, -] - -[package.dependencies] -setuptools = "*" - -[package.extras] -devel = ["coveralls", "flake8 (<3.5.0)", "mock", "nose", "pycodestyle (<2.4.0)", "pylint"] -query = ["jmespath"] -schema = ["jsonschema"] -template = ["Jinja2"] -toml = ["toml"] -yaml = ["pyyaml"] - -[[package]] -name = "arrow" -version = "1.2.3" -description = "Better dates & times for Python" -optional = false -python-versions = ">=3.6" -files = [ - {file = "arrow-1.2.3-py3-none-any.whl", hash = "sha256:5a49ab92e3b7b71d96cd6bfcc4df14efefc9dfa96ea19045815914a6ab6b1fe2"}, - {file = "arrow-1.2.3.tar.gz", hash = "sha256:3934b30ca1b9f292376d9db15b19446088d12ec58629bc3f0da28fd55fb633a1"}, -] - -[package.dependencies] -python-dateutil = ">=2.7.0" - -[[package]] -name = "attrs" -version = "22.2.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.6" -files = [ - {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, - {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, -] - -[package.extras] -cov = ["attrs[tests]", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] -dev = ["attrs[docs,tests]"] -docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"] -tests = ["attrs[tests-no-zope]", "zope.interface"] -tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy (>=0.971,<0.990)", "mypy (>=0.971,<0.990)", "pympler", "pympler", "pytest (>=4.3.0)", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-mypy-plugins", "pytest-xdist[psutil]", "pytest-xdist[psutil]"] - -[[package]] -name = "binaryornot" -version = "0.4.4" -description = "Ultra-lightweight pure Python package to check if a file is binary or text." -optional = false -python-versions = "*" -files = [ - {file = "binaryornot-0.4.4-py2.py3-none-any.whl", hash = "sha256:b8b71173c917bddcd2c16070412e369c3ed7f0528926f70cac18a6c97fd563e4"}, - {file = "binaryornot-0.4.4.tar.gz", hash = "sha256:359501dfc9d40632edc9fac890e19542db1a287bbcfa58175b66658392018061"}, -] - -[package.dependencies] -chardet = ">=3.0.2" - -[[package]] -name = "build" -version = "0.10.0" -description = "A simple, correct Python build frontend" -optional = false -python-versions = ">= 3.7" -files = [ - {file = "build-0.10.0-py3-none-any.whl", hash = "sha256:af266720050a66c893a6096a2f410989eeac74ff9a68ba194b3f6473e8e26171"}, - {file = "build-0.10.0.tar.gz", hash = "sha256:d5b71264afdb5951d6704482aac78de887c80691c52b88a9ad195983ca2c9269"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "os_name == \"nt\""} -packaging = ">=19.0" -pyproject_hooks = "*" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} - -[package.extras] -docs = ["furo (>=2021.08.31)", "sphinx (>=4.0,<5.0)", "sphinx-argparse-cli (>=1.5)", "sphinx-autodoc-typehints (>=1.10)"] -test = ["filelock (>=3)", "pytest (>=6.2.4)", "pytest-cov (>=2.12)", "pytest-mock (>=2)", "pytest-rerunfailures (>=9.1)", "pytest-xdist (>=1.34)", "setuptools (>=42.0.0)", "setuptools (>=56.0.0)", "toml (>=0.10.0)", "wheel (>=0.36.0)"] -typing = ["importlib-metadata (>=5.1)", "mypy (==0.991)", "tomli", "typing-extensions (>=3.7.4.3)"] -virtualenv = ["virtualenv (>=20.0.35)"] - -[[package]] -name = "cachetools" -version = "5.3.0" -description = "Extensible memoizing collections and decorators" -optional = false -python-versions = "~=3.7" -files = [ - {file = "cachetools-5.3.0-py3-none-any.whl", hash = "sha256:429e1a1e845c008ea6c85aa35d4b98b65d6a9763eeef3e37e92728a12d1de9d4"}, - {file = "cachetools-5.3.0.tar.gz", hash = "sha256:13dfddc7b8df938c21a940dfa6557ce6e94a2f1cdfa58eb90c805721d58f2c14"}, -] - -[[package]] -name = "certifi" -version = "2024.8.30" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.6" -files = [ - {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, - {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, -] - -[[package]] -name = "chardet" -version = "5.1.0" -description = "Universal encoding detector for Python 3" -optional = false -python-versions = ">=3.7" -files = [ - {file = "chardet-5.1.0-py3-none-any.whl", hash = "sha256:362777fb014af596ad31334fde1e8c327dfdb076e1960d1694662d46a6917ab9"}, - {file = "chardet-5.1.0.tar.gz", hash = "sha256:0d62712b956bc154f85fb0a266e2a3c5913c2967e00348701b32411d6def31e5"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.0.1" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = "*" -files = [ - {file = "charset-normalizer-3.0.1.tar.gz", hash = "sha256:ebea339af930f8ca5d7a699b921106c6e29c617fe9606fa7baa043c1cdae326f"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88600c72ef7587fe1708fd242b385b6ed4b8904976d5da0893e31df8b3480cb6"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c75ffc45f25324e68ab238cb4b5c0a38cd1c3d7f1fb1f72b5541de469e2247db"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db72b07027db150f468fbada4d85b3b2729a3db39178abf5c543b784c1254539"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62595ab75873d50d57323a91dd03e6966eb79c41fa834b7a1661ed043b2d404d"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff6f3db31555657f3163b15a6b7c6938d08df7adbfc9dd13d9d19edad678f1e8"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:772b87914ff1152b92a197ef4ea40efe27a378606c39446ded52c8f80f79702e"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70990b9c51340e4044cfc394a81f614f3f90d41397104d226f21e66de668730d"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:292d5e8ba896bbfd6334b096e34bffb56161c81408d6d036a7dfa6929cff8783"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2edb64ee7bf1ed524a1da60cdcd2e1f6e2b4f66ef7c077680739f1641f62f555"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:31a9ddf4718d10ae04d9b18801bd776693487cbb57d74cc3458a7673f6f34639"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:44ba614de5361b3e5278e1241fda3dc1838deed864b50a10d7ce92983797fa76"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:12db3b2c533c23ab812c2b25934f60383361f8a376ae272665f8e48b88e8e1c6"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c512accbd6ff0270939b9ac214b84fb5ada5f0409c44298361b2f5e13f9aed9e"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-win32.whl", hash = "sha256:502218f52498a36d6bf5ea77081844017bf7982cdbe521ad85e64cabee1b608b"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:601f36512f9e28f029d9481bdaf8e89e5148ac5d89cffd3b05cd533eeb423b59"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0298eafff88c99982a4cf66ba2efa1128e4ddaca0b05eec4c456bbc7db691d8d"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a8d0fc946c784ff7f7c3742310cc8a57c5c6dc31631269876a88b809dbeff3d3"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:87701167f2a5c930b403e9756fab1d31d4d4da52856143b609e30a1ce7160f3c"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14e76c0f23218b8f46c4d87018ca2e441535aed3632ca134b10239dfb6dadd6b"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c0a590235ccd933d9892c627dec5bc7511ce6ad6c1011fdf5b11363022746c1"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c7fe7afa480e3e82eed58e0ca89f751cd14d767638e2550c77a92a9e749c317"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79909e27e8e4fcc9db4addea88aa63f6423ebb171db091fb4373e3312cb6d603"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ac7b6a045b814cf0c47f3623d21ebd88b3e8cf216a14790b455ea7ff0135d18"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:72966d1b297c741541ca8cf1223ff262a6febe52481af742036a0b296e35fa5a"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f9d0c5c045a3ca9bedfc35dca8526798eb91a07aa7a2c0fee134c6c6f321cbd7"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:5995f0164fa7df59db4746112fec3f49c461dd6b31b841873443bdb077c13cfc"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4a8fcf28c05c1f6d7e177a9a46a1c52798bfe2ad80681d275b10dcf317deaf0b"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:761e8904c07ad053d285670f36dd94e1b6ab7f16ce62b9805c475b7aa1cffde6"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-win32.whl", hash = "sha256:71140351489970dfe5e60fc621ada3e0f41104a5eddaca47a7acb3c1b851d6d3"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:9ab77acb98eba3fd2a85cd160851816bfce6871d944d885febf012713f06659c"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:84c3990934bae40ea69a82034912ffe5a62c60bbf6ec5bc9691419641d7d5c9a"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74292fc76c905c0ef095fe11e188a32ebd03bc38f3f3e9bcb85e4e6db177b7ea"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c95a03c79bbe30eec3ec2b7f076074f4281526724c8685a42872974ef4d36b72"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4c39b0e3eac288fedc2b43055cfc2ca7a60362d0e5e87a637beac5d801ef478"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df2c707231459e8a4028eabcd3cfc827befd635b3ef72eada84ab13b52e1574d"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93ad6d87ac18e2a90b0fe89df7c65263b9a99a0eb98f0a3d2e079f12a0735837"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:59e5686dd847347e55dffcc191a96622f016bc0ad89105e24c14e0d6305acbc6"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:cd6056167405314a4dc3c173943f11249fa0f1b204f8b51ed4bde1a9cd1834dc"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:083c8d17153ecb403e5e1eb76a7ef4babfc2c48d58899c98fcaa04833e7a2f9a"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:f5057856d21e7586765171eac8b9fc3f7d44ef39425f85dbcccb13b3ebea806c"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:7eb33a30d75562222b64f569c642ff3dc6689e09adda43a082208397f016c39a"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:95dea361dd73757c6f1c0a1480ac499952c16ac83f7f5f4f84f0658a01b8ef41"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:eaa379fcd227ca235d04152ca6704c7cb55564116f8bc52545ff357628e10602"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3e45867f1f2ab0711d60c6c71746ac53537f1684baa699f4f668d4c6f6ce8e14"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cadaeaba78750d58d3cc6ac4d1fd867da6fc73c88156b7a3212a3cd4819d679d"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:911d8a40b2bef5b8bbae2e36a0b103f142ac53557ab421dc16ac4aafee6f53dc"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:503e65837c71b875ecdd733877d852adbc465bd82c768a067badd953bf1bc5a3"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a60332922359f920193b1d4826953c507a877b523b2395ad7bc716ddd386d866"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:16a8663d6e281208d78806dbe14ee9903715361cf81f6d4309944e4d1e59ac5b"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:a16418ecf1329f71df119e8a65f3aa68004a3f9383821edcb20f0702934d8087"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9d9153257a3f70d5f69edf2325357251ed20f772b12e593f3b3377b5f78e7ef8"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:02a51034802cbf38db3f89c66fb5d2ec57e6fe7ef2f4a44d070a593c3688667b"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:2e396d70bc4ef5325b72b593a72c8979999aa52fb8bcf03f701c1b03e1166918"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:11b53acf2411c3b09e6af37e4b9005cba376c872503c8f28218c7243582df45d"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:0bf2dae5291758b6f84cf923bfaa285632816007db0330002fa1de38bfcb7154"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2c03cc56021a4bd59be889c2b9257dae13bf55041a3372d3295416f86b295fb5"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:024e606be3ed92216e2b6952ed859d86b4cfa52cd5bc5f050e7dc28f9b43ec42"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4b0d02d7102dd0f997580b51edc4cebcf2ab6397a7edf89f1c73b586c614272c"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:358a7c4cb8ba9b46c453b1dd8d9e431452d5249072e4f56cfda3149f6ab1405e"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81d6741ab457d14fdedc215516665050f3822d3e56508921cc7239f8c8e66a58"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8b8af03d2e37866d023ad0ddea594edefc31e827fee64f8de5611a1dbc373174"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9cf4e8ad252f7c38dd1f676b46514f92dc0ebeb0db5552f5f403509705e24753"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e696f0dd336161fca9adbb846875d40752e6eba585843c768935ba5c9960722b"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c22d3fe05ce11d3671297dc8973267daa0f938b93ec716e12e0f6dee81591dc1"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:109487860ef6a328f3eec66f2bf78b0b72400280d8f8ea05f69c51644ba6521a"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:37f8febc8ec50c14f3ec9637505f28e58d4f66752207ea177c1d67df25da5aed"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f97e83fa6c25693c7a35de154681fcc257c1c41b38beb0304b9c4d2d9e164479"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a152f5f33d64a6be73f1d30c9cc82dfc73cec6477ec268e7c6e4c7d23c2d2291"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:39049da0ffb96c8cbb65cbf5c5f3ca3168990adf3551bd1dee10c48fce8ae820"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-win32.whl", hash = "sha256:4457ea6774b5611f4bed5eaa5df55f70abde42364d498c5134b7ef4c6958e20e"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:e62164b50f84e20601c1ff8eb55620d2ad25fb81b59e3cd776a1902527a788af"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8eade758719add78ec36dc13201483f8e9b5d940329285edcd5f70c0a9edbd7f"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8499ca8f4502af841f68135133d8258f7b32a53a1d594aa98cc52013fff55678"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3fc1c4a2ffd64890aebdb3f97e1278b0cc72579a08ca4de8cd2c04799a3a22be"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00d3ffdaafe92a5dc603cb9bd5111aaa36dfa187c8285c543be562e61b755f6b"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2ac1b08635a8cd4e0cbeaf6f5e922085908d48eb05d44c5ae9eabab148512ca"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6f45710b4459401609ebebdbcfb34515da4fc2aa886f95107f556ac69a9147e"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ae1de54a77dc0d6d5fcf623290af4266412a7c4be0b1ff7444394f03f5c54e3"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b590df687e3c5ee0deef9fc8c547d81986d9a1b56073d82de008744452d6541"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab5de034a886f616a5668aa5d098af2b5385ed70142090e2a31bcbd0af0fdb3d"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9cb3032517f1627cc012dbc80a8ec976ae76d93ea2b5feaa9d2a5b8882597579"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:608862a7bf6957f2333fc54ab4399e405baad0163dc9f8d99cb236816db169d4"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0f438ae3532723fb6ead77e7c604be7c8374094ef4ee2c5e03a3a17f1fca256c"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:356541bf4381fa35856dafa6a965916e54bed415ad8a24ee6de6e37deccf2786"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-win32.whl", hash = "sha256:39cf9ed17fe3b1bc81f33c9ceb6ce67683ee7526e65fde1447c772afc54a1bb8"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0a11e971ed097d24c534c037d298ad32c6ce81a45736d31e0ff0ad37ab437d59"}, - {file = "charset_normalizer-3.0.1-py3-none-any.whl", hash = "sha256:7e189e2e1d3ed2f4aebabd2d5b0f931e883676e51c7624826e0a4e5fe8a0bf24"}, -] - -[[package]] -name = "click" -version = "8.1.3" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.7" -files = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "cookiecutter" -version = "2.1.1" -description = "A command-line utility that creates projects from project templates, e.g. creating a Python package project from a Python package project template." -optional = false -python-versions = ">=3.7" -files = [ - {file = "cookiecutter-2.1.1-py2.py3-none-any.whl", hash = "sha256:9f3ab027cec4f70916e28f03470bdb41e637a3ad354b4d65c765d93aad160022"}, - {file = "cookiecutter-2.1.1.tar.gz", hash = "sha256:f3982be8d9c53dac1261864013fdec7f83afd2e42ede6f6dd069c5e149c540d5"}, -] - -[package.dependencies] -binaryornot = ">=0.4.4" -click = ">=7.0,<9.0.0" -Jinja2 = ">=2.7,<4.0.0" -jinja2-time = ">=0.2.0" -python-slugify = ">=4.0.0" -pyyaml = ">=5.3.1" -requests = ">=2.23.0" - -[[package]] -name = "dynaconf" -version = "3.1.11" -description = "The dynamic configurator for your Python Project" -optional = false -python-versions = ">=3.7" -files = [ - {file = "dynaconf-3.1.11-py2.py3-none-any.whl", hash = "sha256:87e0b3b12b5db9e8fb465e1f8c7fdb926cd2ec5b6d88aa7f821f316df93fb165"}, - {file = "dynaconf-3.1.11.tar.gz", hash = "sha256:d9cfb50fd4a71a543fd23845d4f585b620b6ff6d9d3cc1825c614f7b2097cb39"}, -] - -[package.extras] -all = ["configobj", "hvac", "redis", "ruamel.yaml"] -configobj = ["configobj"] -ini = ["configobj"] -redis = ["redis"] -test = ["codecov", "configobj", "django", "flake8", "flake8-debugger", "flake8-print", "flake8-todo", "flask (>=0.12)", "hvac", "pep8-naming", "pytest", "pytest-cov", "pytest-mock", "pytest-xdist", "python-dotenv", "radon", "redis", "toml"] -toml = ["toml"] -vault = ["hvac"] -yaml = ["ruamel.yaml"] - -[[package]] -name = "fsspec" -version = "2023.1.0" -description = "File-system specification" -optional = false -python-versions = ">=3.7" -files = [ - {file = "fsspec-2023.1.0-py3-none-any.whl", hash = "sha256:b833e2e541e9e8cde0ab549414187871243177feb3d344f9d27b25a93f5d8139"}, - {file = "fsspec-2023.1.0.tar.gz", hash = "sha256:fbae7f20ff801eb5f7d0bedf81f25c787c0dfac5e982d98fa3884a9cde2b5411"}, -] - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -entrypoints = ["importlib-metadata"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -tqdm = ["tqdm"] - -[[package]] -name = "gitdb" -version = "4.0.10" -description = "Git Object Database" -optional = false -python-versions = ">=3.7" -files = [ - {file = "gitdb-4.0.10-py3-none-any.whl", hash = "sha256:c286cf298426064079ed96a9e4a9d39e7f3e9bf15ba60701e95f5492f28415c7"}, - {file = "gitdb-4.0.10.tar.gz", hash = "sha256:6eb990b69df4e15bad899ea868dc46572c3f75339735663b81de79b06f17eb9a"}, -] - -[package.dependencies] -smmap = ">=3.0.1,<6" - -[[package]] -name = "gitpython" -version = "3.1.43" -description = "GitPython is a Python library used to interact with Git repositories" -optional = false -python-versions = ">=3.7" -files = [ - {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, - {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, -] - -[package.dependencies] -gitdb = ">=4.0.1,<5" - -[package.extras] -doc = ["sphinx (==4.3.2)", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "sphinxcontrib-applehelp (>=1.0.2,<=1.0.4)", "sphinxcontrib-devhelp (==1.0.2)", "sphinxcontrib-htmlhelp (>=2.0.0,<=2.0.1)", "sphinxcontrib-qthelp (==1.0.3)", "sphinxcontrib-serializinghtml (==1.1.5)"] -test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] - -[[package]] -name = "idna" -version = "3.4" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.5" -files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, -] - -[[package]] -name = "importlib-metadata" -version = "6.0.0" -description = "Read metadata from Python packages" -optional = false -python-versions = ">=3.7" -files = [ - {file = "importlib_metadata-6.0.0-py3-none-any.whl", hash = "sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad"}, - {file = "importlib_metadata-6.0.0.tar.gz", hash = "sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d"}, -] - -[package.dependencies] -zipp = ">=0.5" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -perf = ["ipython"] -testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] - -[[package]] -name = "importlib-resources" -version = "5.12.0" -description = "Read resources from Python packages" -optional = false -python-versions = ">=3.7" -files = [ - {file = "importlib_resources-5.12.0-py3-none-any.whl", hash = "sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a"}, - {file = "importlib_resources-5.12.0.tar.gz", hash = "sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6"}, -] - -[package.dependencies] -zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - -[[package]] -name = "jinja2" -version = "3.1.2" -description = "A very fast and expressive template engine." -optional = false -python-versions = ">=3.7" -files = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, -] - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "jinja2-time" -version = "0.2.0" -description = "Jinja2 Extension for Dates and Times" -optional = false -python-versions = "*" -files = [ - {file = "jinja2-time-0.2.0.tar.gz", hash = "sha256:d14eaa4d315e7688daa4969f616f226614350c48730bfa1692d2caebd8c90d40"}, - {file = "jinja2_time-0.2.0-py2.py3-none-any.whl", hash = "sha256:d3eab6605e3ec8b7a0863df09cc1d23714908fa61aa6986a845c20ba488b4efa"}, -] - -[package.dependencies] -arrow = "*" -jinja2 = "*" - -[[package]] -name = "jmespath" -version = "0.10.0" -description = "JSON Matching Expressions" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"}, - {file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"}, -] - -[[package]] -name = "kedro" -version = "0.18.5" -description = "Kedro helps you build production-ready data and analytics pipelines" -optional = false -python-versions = ">=3.7, <3.11" -files = [ - {file = "kedro-0.18.5-py3-none-any.whl", hash = "sha256:bf1c24816e1ff383ee1a07eac8866f1fe0d82703bf42ca1a376869d59eb177cd"}, - {file = "kedro-0.18.5.tar.gz", hash = "sha256:eb972b4b78b980e621837e59af72083f9fcc8fe4da8dd70d1e1d5ba66df790f8"}, -] - -[package.dependencies] -anyconfig = ">=0.10.0,<0.11.0" -attrs = ">=21.3" -cachetools = ">=5.3,<6.0" -click = "<9.0" -cookiecutter = ">=2.1.1,<3.0" -dynaconf = ">=3.1.2,<4.0" -fsspec = ">=2021.4,<2023.1.1" -gitpython = ">=3.0,<4.0" -importlib-metadata = {version = ">=3.6", markers = "python_version >= \"3.8\""} -importlib-resources = ">=1.3" -jmespath = ">=0.9.5,<1.0" -more-itertools = ">=9.0,<10.0" -omegaconf = ">=2.3,<3.0" -pip-tools = ">=6.12,<7.0" -pluggy = ">=1.0.0,<1.1.0" -PyYAML = ">=4.2,<7.0" -rich = ">=13.3,<14.0" -rope = ">=1.7.0,<1.8.0" -setuptools = ">=65.5.1" -toml = ">=0.10,<1.0" -toposort = ">=1.9,<2.0" - -[package.extras] -all = ["Jinja2 (<3.1.0)", "Pillow (>=9.0,<10.0)", "PyYAML (>=4.2,<7.0)", "SQLAlchemy (>=1.2,<2.0)", "biopython (>=1.73,<2.0)", "compress-pickle[lz4] (>=2.1.0,<2.2.0)", "dask[complete] (>=2021.10,<2022.0)", "delta-spark (>=1.0,<3.0)", "docutils (==0.16)", "geopandas (>=0.6.0,<1.0)", "hdfs (>=2.5.8,<3.0)", "holoviews (>=1.13.0,<1.14.0)", "ipykernel (>=5.3,<7.0)", "lxml (>=4.6,<5.0)", "matplotlib (>=3.0.3,<4.0)", "myst-parser (>=0.17.2,<0.18.0)", "nbsphinx (==0.8.1)", "nbstripout (>=0.4,<1.0)", "networkx (>=2.4,<3.0)", "opencv-python (>=4.5.5.64,<4.6.0.0)", "openpyxl (>=3.0.6,<4.0)", "pandas (>=1.3,<2.0)", "pandas-gbq (>=0.12.0,<0.18.0)", "plotly (>=4.8.0,<6.0)", "pyarrow (>=1.0,<7.0)", "pyproj (>=3.0,<4.0)", "pyspark (>=2.2,<4.0)", "redis (>=4.1,<5.0)", "requests (>=2.20,<3.0)", "s3fs (>=0.3.0,<0.5)", "scikit-learn (>=1.0.2,<1.1.0)", "scipy (>=1.7.3,<1.8.0)", "sphinx (>=3.4.3,<3.5.0)", "sphinx-autodoc-typehints (==1.11.1)", "sphinx-copybutton (==0.3.1)", "sphinx-rtd-theme (==1.1.1)", "sphinxcontrib-mermaid (>=0.7.1,<0.8.0)", "tables (>=3.6,<4.0)", "tables (>=3.6.0,<3.7.0)", "tensorflow (>=2.0,<3.0)", "triad (>=0.6.7,<1.0)"] -api = ["requests (>=2.20,<3.0)"] -api-apidataset = ["requests (>=2.20,<3.0)"] -biosequence = ["biopython (>=1.73,<2.0)"] -biosequence-biosequencedataset = ["biopython (>=1.73,<2.0)"] -dask = ["dask[complete] (>=2021.10,<2022.0)", "triad (>=0.6.7,<1.0)"] -dask-parquetdataset = ["dask[complete] (>=2021.10,<2022.0)", "triad (>=0.6.7,<1.0)"] -docs = ["Jinja2 (<3.1.0)", "docutils (==0.16)", "ipykernel (>=5.3,<7.0)", "myst-parser (>=0.17.2,<0.18.0)", "nbsphinx (==0.8.1)", "nbstripout (>=0.4,<1.0)", "sphinx (>=3.4.3,<3.5.0)", "sphinx-autodoc-typehints (==1.11.1)", "sphinx-copybutton (==0.3.1)", "sphinx-rtd-theme (==1.1.1)", "sphinxcontrib-mermaid (>=0.7.1,<0.8.0)"] -geopandas = ["geopandas (>=0.6.0,<1.0)", "pyproj (>=3.0,<4.0)"] -geopandas-geojsondataset = ["geopandas (>=0.6.0,<1.0)", "pyproj (>=3.0,<4.0)"] -holoviews = ["holoviews (>=1.13.0,<1.14.0)"] -holoviews-holoviewswriter = ["holoviews (>=1.13.0,<1.14.0)"] -matplotlib = ["matplotlib (>=3.0.3,<4.0)"] -matplotlib-matplotlibwriter = ["matplotlib (>=3.0.3,<4.0)"] -networkx = ["networkx (>=2.4,<3.0)"] -networkx-networkxdataset = ["networkx (>=2.4,<3.0)"] -pandas = ["SQLAlchemy (>=1.2,<2.0)", "lxml (>=4.6,<5.0)", "openpyxl (>=3.0.6,<4.0)", "pandas (>=1.3,<2.0)", "pandas-gbq (>=0.12.0,<0.18.0)", "pyarrow (>=1.0,<7.0)", "tables (>=3.6,<4.0)", "tables (>=3.6.0,<3.7.0)"] -pandas-csvdataset = ["pandas (>=1.3,<2.0)"] -pandas-exceldataset = ["openpyxl (>=3.0.6,<4.0)", "pandas (>=1.3,<2.0)"] -pandas-featherdataset = ["pandas (>=1.3,<2.0)"] -pandas-gbqquerydataset = ["pandas (>=1.3,<2.0)", "pandas-gbq (>=0.12.0,<0.18.0)"] -pandas-gbqtabledataset = ["pandas (>=1.3,<2.0)", "pandas-gbq (>=0.12.0,<0.18.0)"] -pandas-genericdataset = ["pandas (>=1.3,<2.0)"] -pandas-hdfdataset = ["pandas (>=1.3,<2.0)", "tables (>=3.6,<4.0)", "tables (>=3.6.0,<3.7.0)"] -pandas-jsondataset = ["pandas (>=1.3,<2.0)"] -pandas-parquetdataset = ["pandas (>=1.3,<2.0)", "pyarrow (>=1.0,<7.0)"] -pandas-sqlquerydataset = ["SQLAlchemy (>=1.2,<2.0)", "pandas (>=1.3,<2.0)"] -pandas-sqltabledataset = ["SQLAlchemy (>=1.2,<2.0)", "pandas (>=1.3,<2.0)"] -pandas-xmldataset = ["lxml (>=4.6,<5.0)", "pandas (>=1.3,<2.0)"] -pickle = ["compress-pickle[lz4] (>=2.1.0,<2.2.0)"] -pickle-pickledataset = ["compress-pickle[lz4] (>=2.1.0,<2.2.0)"] -pillow = ["Pillow (>=9.0,<10.0)"] -pillow-imagedataset = ["Pillow (>=9.0,<10.0)"] -plotly = ["pandas (>=1.3,<2.0)", "plotly (>=4.8.0,<6.0)"] -plotly-jsondataset = ["plotly (>=4.8.0,<6.0)"] -plotly-plotlydataset = ["pandas (>=1.3,<2.0)", "plotly (>=4.8.0,<6.0)"] -redis = ["redis (>=4.1,<5.0)"] -spark = ["delta-spark (>=1.0,<3.0)", "hdfs (>=2.5.8,<3.0)", "pyspark (>=2.2,<4.0)", "s3fs (>=0.3.0,<0.5)"] -spark-deltatabledataset = ["delta-spark (>=1.0,<3.0)", "hdfs (>=2.5.8,<3.0)", "pyspark (>=2.2,<4.0)", "s3fs (>=0.3.0,<0.5)"] -spark-sparkdataset = ["hdfs (>=2.5.8,<3.0)", "pyspark (>=2.2,<4.0)", "s3fs (>=0.3.0,<0.5)"] -spark-sparkhivedataset = ["hdfs (>=2.5.8,<3.0)", "pyspark (>=2.2,<4.0)", "s3fs (>=0.3.0,<0.5)"] -spark-sparkjdbcdataset = ["hdfs (>=2.5.8,<3.0)", "pyspark (>=2.2,<4.0)", "s3fs (>=0.3.0,<0.5)"] -svmlight = ["scikit-learn (>=1.0.2,<1.1.0)", "scipy (>=1.7.3,<1.8.0)"] -svmlight-svmlightdataset = ["scikit-learn (>=1.0.2,<1.1.0)", "scipy (>=1.7.3,<1.8.0)"] -tensorflow = ["tensorflow (>=2.0,<3.0)"] -tensorflow-tensorflowmodeldataset = ["tensorflow (>=2.0,<3.0)"] -video = ["opencv-python (>=4.5.5.64,<4.6.0.0)"] -video-videodataset = ["opencv-python (>=4.5.5.64,<4.6.0.0)"] -yaml = ["PyYAML (>=4.2,<7.0)", "pandas (>=1.3,<2.0)"] -yaml-yamldataset = ["PyYAML (>=4.2,<7.0)", "pandas (>=1.3,<2.0)"] - -[[package]] -name = "markdown-it-py" -version = "2.2.0" -description = "Python port of markdown-it. Markdown parsing, done right!" -optional = false -python-versions = ">=3.7" -files = [ - {file = "markdown-it-py-2.2.0.tar.gz", hash = "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"}, - {file = "markdown_it_py-2.2.0-py3-none-any.whl", hash = "sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30"}, -] - -[package.dependencies] -mdurl = ">=0.1,<1.0" - -[package.extras] -benchmarking = ["psutil", "pytest", "pytest-benchmark"] -code-style = ["pre-commit (>=3.0,<4.0)"] -compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] -linkify = ["linkify-it-py (>=1,<3)"] -plugins = ["mdit-py-plugins"] -profiling = ["gprof2dot"] -rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] -testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] - -[[package]] -name = "markupsafe" -version = "2.1.2" -description = "Safely add untrusted strings to HTML/XML markup." -optional = false -python-versions = ">=3.7" -files = [ - {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-win32.whl", hash = "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-win32.whl", hash = "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-win32.whl", hash = "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-win32.whl", hash = "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-win32.whl", hash = "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed"}, - {file = "MarkupSafe-2.1.2.tar.gz", hash = "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d"}, -] - -[[package]] -name = "mdurl" -version = "0.1.2" -description = "Markdown URL utilities" -optional = false -python-versions = ">=3.7" -files = [ - {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, - {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, -] - -[[package]] -name = "more-itertools" -version = "9.1.0" -description = "More routines for operating on iterables, beyond itertools" -optional = false -python-versions = ">=3.7" -files = [ - {file = "more-itertools-9.1.0.tar.gz", hash = "sha256:cabaa341ad0389ea83c17a94566a53ae4c9d07349861ecb14dc6d0345cf9ac5d"}, - {file = "more_itertools-9.1.0-py3-none-any.whl", hash = "sha256:d2bc7f02446e86a68911e58ded76d6561eea00cddfb2a91e7019bbb586c799f3"}, -] - -[[package]] -name = "omegaconf" -version = "2.3.0" -description = "A flexible configuration library" -optional = false -python-versions = ">=3.6" -files = [ - {file = "omegaconf-2.3.0-py3-none-any.whl", hash = "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b"}, - {file = "omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7"}, -] - -[package.dependencies] -antlr4-python3-runtime = "==4.9.*" -PyYAML = ">=5.1.0" - -[[package]] -name = "packaging" -version = "23.0" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.7" -files = [ - {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, - {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, -] - -[[package]] -name = "pip" -version = "23.0.1" -description = "The PyPA recommended tool for installing Python packages." -optional = false -python-versions = ">=3.7" -files = [ - {file = "pip-23.0.1-py3-none-any.whl", hash = "sha256:236bcb61156d76c4b8a05821b988c7b8c35bf0da28a4b614e8d6ab5212c25c6f"}, - {file = "pip-23.0.1.tar.gz", hash = "sha256:cd015ea1bfb0fcef59d8a286c1f8bebcb983f6317719d415dc5351efb7cd7024"}, -] - -[[package]] -name = "pip-tools" -version = "6.12.2" -description = "pip-tools keeps your pinned dependencies fresh." -optional = false -python-versions = ">=3.7" -files = [ - {file = "pip-tools-6.12.2.tar.gz", hash = "sha256:8b903696df4598b10d469026ef9995c5f9a874b416e88e7a214884ebe4a70245"}, - {file = "pip_tools-6.12.2-py3-none-any.whl", hash = "sha256:6a51f4fd67140d5e83703ebfa9610fb61398727151f56a1be02a972d062e4679"}, -] - -[package.dependencies] -build = "*" -click = ">=8" -pip = ">=22.2" -setuptools = "*" -wheel = "*" - -[package.extras] -coverage = ["pytest-cov"] -testing = ["flit-core (>=2,<4)", "poetry-core (>=1.0.0)", "pytest (>=7.2.0)", "pytest-rerunfailures", "pytest-xdist"] - -[[package]] -name = "platformdirs" -version = "3.0.0" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -optional = false -python-versions = ">=3.7" -files = [ - {file = "platformdirs-3.0.0-py3-none-any.whl", hash = "sha256:b1d5eb14f221506f50d6604a561f4c5786d9e80355219694a1b244bcd96f4567"}, - {file = "platformdirs-3.0.0.tar.gz", hash = "sha256:8a1228abb1ef82d788f74139988b137e78692984ec7b08eaa6c65f1723af28f9"}, -] - -[package.extras] -docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] - -[[package]] -name = "pluggy" -version = "1.0.0" -description = "plugin and hook calling mechanisms for python" -optional = false -python-versions = ">=3.6" -files = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, -] - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "pygments" -version = "2.14.0" -description = "Pygments is a syntax highlighting package written in Python." -optional = false -python-versions = ">=3.6" -files = [ - {file = "Pygments-2.14.0-py3-none-any.whl", hash = "sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717"}, - {file = "Pygments-2.14.0.tar.gz", hash = "sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297"}, -] - -[package.extras] -plugins = ["importlib-metadata"] - -[[package]] -name = "pyproject-hooks" -version = "1.0.0" -description = "Wrappers to call pyproject.toml-based build backend hooks." -optional = false -python-versions = ">=3.7" -files = [ - {file = "pyproject_hooks-1.0.0-py3-none-any.whl", hash = "sha256:283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8"}, - {file = "pyproject_hooks-1.0.0.tar.gz", hash = "sha256:f271b298b97f5955d53fb12b72c1fb1948c22c1a6b70b315c54cedaca0264ef5"}, -] - -[package.dependencies] -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "python-slugify" -version = "8.0.1" -description = "A Python slugify application that also handles Unicode" -optional = false -python-versions = ">=3.7" -files = [ - {file = "python-slugify-8.0.1.tar.gz", hash = "sha256:ce0d46ddb668b3be82f4ed5e503dbc33dd815d83e2eb6824211310d3fb172a27"}, - {file = "python_slugify-8.0.1-py2.py3-none-any.whl", hash = "sha256:70ca6ea68fe63ecc8fa4fcf00ae651fc8a5d02d93dcd12ae6d4fc7ca46c4d395"}, -] - -[package.dependencies] -text-unidecode = ">=1.3" - -[package.extras] -unidecode = ["Unidecode (>=1.1.1)"] - -[[package]] -name = "pytoolconfig" -version = "1.2.5" -description = "Python tool configuration" -optional = false -python-versions = ">=3.7" -files = [ - {file = "pytoolconfig-1.2.5-py3-none-any.whl", hash = "sha256:239ba9d3e537b91d0243275a497700ea39a5e259ddb80421c366e3b288bf30fe"}, - {file = "pytoolconfig-1.2.5.tar.gz", hash = "sha256:a50f9dfe23b03a9d40414c1fdf902fefbeae12f2ac75a3c8f915944d6ffac279"}, -] - -[package.dependencies] -packaging = ">=22.0" -platformdirs = {version = ">=1.4.4", optional = true, markers = "extra == \"global\""} -tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} - -[package.extras] -doc = ["sphinx (>=4.5.0)", "tabulate (>=0.8.9)"] -gendocs = ["pytoolconfig[doc]", "sphinx (>=4.5.0)", "sphinx-autodoc-typehints (>=1.18.1)", "sphinx-rtd-theme (>=1.0.0)"] -global = ["platformdirs (>=1.4.4)"] -validation = ["pydantic (>=1.7.4)"] - -[[package]] -name = "pyyaml" -version = "6.0" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.6" -files = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, -] - -[[package]] -name = "requests" -version = "2.28.2" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.7, <4" -files = [ - {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, - {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, -] - -[package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<1.27" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] - -[[package]] -name = "rich" -version = "13.3.1" -description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -optional = false -python-versions = ">=3.7.0" -files = [ - {file = "rich-13.3.1-py3-none-any.whl", hash = "sha256:8aa57747f3fc3e977684f0176a88e789be314a99f99b43b75d1e9cb5dc6db9e9"}, - {file = "rich-13.3.1.tar.gz", hash = "sha256:125d96d20c92b946b983d0d392b84ff945461e5a06d3867e9f9e575f8697b67f"}, -] - -[package.dependencies] -markdown-it-py = ">=2.1.0,<3.0.0" -pygments = ">=2.14.0,<3.0.0" - -[package.extras] -jupyter = ["ipywidgets (>=7.5.1,<9)"] - -[[package]] -name = "rope" -version = "1.7.0" -description = "a python refactoring library..." -optional = false -python-versions = ">=3.7" -files = [ - {file = "rope-1.7.0-py3-none-any.whl", hash = "sha256:893dd80ba7077fc9f6f42b0a849372076b70f1d6e405b9f0cc52781ffa0e6890"}, - {file = "rope-1.7.0.tar.gz", hash = "sha256:ba39581d0f8dee4ae8b5b5e82e35d03cebad965ccb127b7eaab9755cdc85e85a"}, -] - -[package.dependencies] -pytoolconfig = {version = ">=1.2.2", extras = ["global"]} - -[package.extras] -dev = ["build (>=0.7.0)", "pre-commit (>=2.20.0)", "pytest (>=7.0.1)", "pytest-timeout (>=2.1.0)"] -doc = ["pytoolconfig[doc]", "sphinx (>=4.5.0)", "sphinx-autodoc-typehints (>=1.18.1)", "sphinx-rtd-theme (>=1.0.0)"] - -[[package]] -name = "setuptools" -version = "67.4.0" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -optional = false -python-versions = ">=3.7" -files = [ - {file = "setuptools-67.4.0-py3-none-any.whl", hash = "sha256:f106dee1b506dee5102cc3f3e9e68137bbad6d47b616be7991714b0c62204251"}, - {file = "setuptools-67.4.0.tar.gz", hash = "sha256:e5fd0a713141a4a105412233c63dc4e17ba0090c8e8334594ac790ec97792330"}, -] - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] - -[[package]] -name = "smmap" -version = "5.0.0" -description = "A pure Python implementation of a sliding window memory map manager" -optional = false -python-versions = ">=3.6" -files = [ - {file = "smmap-5.0.0-py3-none-any.whl", hash = "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94"}, - {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, -] - -[[package]] -name = "text-unidecode" -version = "1.3" -description = "The most basic Text::Unidecode port" -optional = false -python-versions = "*" -files = [ - {file = "text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93"}, - {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"}, -] - -[[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, -] - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] - -[[package]] -name = "toposort" -version = "1.10" -description = "Implements a topological sort algorithm." -optional = false -python-versions = "*" -files = [ - {file = "toposort-1.10-py3-none-any.whl", hash = "sha256:cbdbc0d0bee4d2695ab2ceec97fe0679e9c10eab4b2a87a9372b929e70563a87"}, - {file = "toposort-1.10.tar.gz", hash = "sha256:bfbb479c53d0a696ea7402601f4e693c97b0367837c8898bc6471adfca37a6bd"}, -] - -[[package]] -name = "urllib3" -version = "1.26.14" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" -files = [ - {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, - {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, -] - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] - -[[package]] -name = "wheel" -version = "0.38.4" -description = "A built-package format for Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "wheel-0.38.4-py3-none-any.whl", hash = "sha256:b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8"}, - {file = "wheel-0.38.4.tar.gz", hash = "sha256:965f5259b566725405b05e7cf774052044b1ed30119b5d586b2703aafe8719ac"}, -] - -[package.extras] -test = ["pytest (>=3.0.0)"] - -[[package]] -name = "zipp" -version = "3.15.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -optional = false -python-versions = ">=3.7" -files = [ - {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, - {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, -] - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - -[metadata] -lock-version = "2.0" -python-versions = "^3.9,<3.11" -content-hash = "161eca8b495750e018c6606f344f9bde2b94a4a83d22c7d285ba2a6c18dff803" diff --git a/10_integrations/kedro_modal/pyproject.toml b/10_integrations/kedro_modal/pyproject.toml deleted file mode 100644 index 64901c750..000000000 --- a/10_integrations/kedro_modal/pyproject.toml +++ /dev/null @@ -1,22 +0,0 @@ -[tool.poetry] -name = "kedro-modal" -version = "0.1.0" -description = "" -authors = ["Elias Freider "] -readme = "README.md" -packages = [{include = "kedro_modal"}] - -[tool.poetry.dependencies] -python = "^3.9,<3.11" -kedro = "^0.18.3" -click = "^8.1.3" - - -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" - -[tool.poetry.plugins] - -[tool.poetry.plugins."kedro.project_commands"] -kedro-modal = "kedro_modal.cli:commands" From 5be511fb20f704d3ecd6ec211a651fce57132438 Mon Sep 17 00:00:00 2001 From: Elias Freider Date: Fri, 13 Dec 2024 14:47:28 +0100 Subject: [PATCH 20/20] Mini fixes --- 10_integrations/dbt/dbt_duckdb.py | 2 +- 13_sandboxes/simple_code_interpreter.py | 1 + internal/run_example.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/10_integrations/dbt/dbt_duckdb.py b/10_integrations/dbt/dbt_duckdb.py index 3aa3b41f6..33872033c 100644 --- a/10_integrations/dbt/dbt_duckdb.py +++ b/10_integrations/dbt/dbt_duckdb.py @@ -68,7 +68,7 @@ .add_local_file( LOCAL_DBT_PROJECT / "profiles.yml", remote_path=f"{PROFILES_PATH}/profiles.yml", - ), + ) ) app = modal.App(name="example-dbt-duckdb-s3", image=dbt_image) diff --git a/13_sandboxes/simple_code_interpreter.py b/13_sandboxes/simple_code_interpreter.py index 8f870012d..725b726f9 100644 --- a/13_sandboxes/simple_code_interpreter.py +++ b/13_sandboxes/simple_code_interpreter.py @@ -23,6 +23,7 @@ from typing import Any import modal +import modal.container_process def driver_program(): diff --git a/internal/run_example.py b/internal/run_example.py index 88969d90e..310a1f394 100644 --- a/internal/run_example.py +++ b/internal/run_example.py @@ -15,7 +15,7 @@ def run_script(example): try: print(f"cli args: {example.cli_args}") process = subprocess.run( - map(str, example.cli_args), + [str(x) for x in example.cli_args], env=os.environ | example.env | {"MODAL_SERVE_TIMEOUT": "5.0"}, timeout=TIMEOUT, )