From c8bb000a5f7e0ee32d4fe19ffd019482be3ea786 Mon Sep 17 00:00:00 2001 From: Rahul Maurya Date: Tue, 2 Apr 2024 04:56:02 +0530 Subject: [PATCH 1/5] CPU option in Docker --- Dockerfile | 15 +++++++--- environment_cpu.yml | 37 +++++++++++++++++++++++++ environment.yml => environment_cuda.yml | 5 ++++ 3 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 environment_cpu.yml rename environment.yml => environment_cuda.yml (91%) diff --git a/Dockerfile b/Dockerfile index 3f58873a..60fb335c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,8 @@ FROM ubuntu:latest - ENV CONDA_ENV_NAME=graph ENV PYTHON_VERSION=3.10 - # Basic setup RUN apt update && apt install -y bash \ build-essential \ @@ -24,7 +22,15 @@ ENV PATH="/conda/bin:${PATH}" RUN git clone https://github.com/openclimatefix/graph_weather.git && mv graph_weather/ gw/ && cd gw/ && mv * .. && rm -rf gw/ -RUN conda update -n base -c defaults conda && conda env create -f environment.yml +# Copy the appropriate environment file based on CUDA availability +COPY environment_cpu.yml /tmp/environment_cpu.yml +COPY environment_cuda.yml /tmp/environment_cuda.yml + +RUN conda update -n base -c defaults conda + +# Check if CUDA is available and accordingly choose env +RUN cuda=$(command -v nvcc > /dev/null && echo "true" || echo "false") \ + && if [ "$cuda" == "true" ]; then conda env create -f /tmp/environment_cuda.yml; else conda env create -f /tmp/environment_cpu.yml; fi # Switch to bash shell SHELL ["/bin/bash", "-c"] @@ -35,8 +41,9 @@ RUN echo "source activate ${CONDA_ENV_NAME}" >> ~/.bashrc # Cp in the development directory and install RUN source activate ${CONDA_ENV_NAME} && pip install -e . + # Make RUN commands use the new environment: SHELL ["conda", "run", "-n", "graph", "/bin/bash", "-c"] -# Example commnad that can be used, need to set API_KEY, API_SECRET and SAVE_DIR +# Example command that can be used, need to set API_KEY, API_SECRET and SAVE_DIR CMD ["conda", "run", "-n", "graph", "python", "-u", "train/pl_graph_weather.py", "--gpus", "16", "--hidden", "64", "--num-blocks", "3", "--batch", "16"] diff --git a/environment_cpu.yml b/environment_cpu.yml new file mode 100644 index 00000000..b4e6fa91 --- /dev/null +++ b/environment_cpu.yml @@ -0,0 +1,37 @@ +name: graph +channels: + - pytorch + - pyg + - conda-forge + - defaults +dependencies: + - numcodecs + - pandas + - pip + - pyg + - python=3.11 + - pytorch + - pytorch-cluster + - pytorch-scatter + - pytorch-sparse + - pytorch-spline-conv + - scikit-learn + - scipy + - torchvision + - tqdm + - xarray + - zarr + - h3-py + - numpy + - torch==2.2.2+cpu -f https://download.pytorch.org/whl/torch_stable.html + - pip: + - datasets + - einops + - fsspec + - torch-geometric-temporal + - huggingface-hub + - pysolar + - pytorch-lightning + - click + + diff --git a/environment.yml b/environment_cuda.yml similarity index 91% rename from environment.yml rename to environment_cuda.yml index f2b6478a..ef634aa2 100644 --- a/environment.yml +++ b/environment_cuda.yml @@ -24,6 +24,8 @@ dependencies: - xarray - zarr - h3-py + - numpy + - torch - pip: - datasets - einops @@ -32,3 +34,6 @@ dependencies: - huggingface-hub - pysolar - pytorch-lightning + - click + + \ No newline at end of file From ca318f7c012499f9d1eef920b071f91e24060e6c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 23:30:03 +0000 Subject: [PATCH 2/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- environment_cpu.yml | 2 -- environment_cuda.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/environment_cpu.yml b/environment_cpu.yml index b4e6fa91..7f386a77 100644 --- a/environment_cpu.yml +++ b/environment_cpu.yml @@ -33,5 +33,3 @@ dependencies: - pysolar - pytorch-lightning - click - - diff --git a/environment_cuda.yml b/environment_cuda.yml index ef634aa2..f34ad862 100644 --- a/environment_cuda.yml +++ b/environment_cuda.yml @@ -35,5 +35,3 @@ dependencies: - pysolar - pytorch-lightning - click - - \ No newline at end of file From 5b13575f4434228dd630766911b364ca69e08d84 Mon Sep 17 00:00:00 2001 From: Rahul Maurya Date: Tue, 2 Apr 2024 14:22:36 +0530 Subject: [PATCH 3/5] updated docker --- Dockerfile | 2 +- environment_cpu.yml | 5 ++--- environment_cuda.yml | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 60fb335c..82b493d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:latest ENV CONDA_ENV_NAME=graph -ENV PYTHON_VERSION=3.10 +ENV PYTHON_VERSION=3.12 # Basic setup RUN apt update && apt install -y bash \ diff --git a/environment_cpu.yml b/environment_cpu.yml index b4e6fa91..0ce7a285 100644 --- a/environment_cpu.yml +++ b/environment_cpu.yml @@ -9,8 +9,8 @@ dependencies: - pandas - pip - pyg - - python=3.11 - - pytorch + - python=3.12 + - pytorch cpuonly -c pytorch - pytorch-cluster - pytorch-scatter - pytorch-sparse @@ -23,7 +23,6 @@ dependencies: - zarr - h3-py - numpy - - torch==2.2.2+cpu -f https://download.pytorch.org/whl/torch_stable.html - pip: - datasets - einops diff --git a/environment_cuda.yml b/environment_cuda.yml index ef634aa2..df4088bf 100644 --- a/environment_cuda.yml +++ b/environment_cuda.yml @@ -11,7 +11,7 @@ dependencies: - pandas - pip - pyg - - python=3.11 + - python=3.12 - pytorch - pytorch-cluster - pytorch-scatter From 333395677159d7287d3e5c949f099d8e3573ee40 Mon Sep 17 00:00:00 2001 From: Rahul Maurya <98907006+rahul-maurya11b@users.noreply.github.com> Date: Tue, 2 Apr 2024 15:02:07 +0530 Subject: [PATCH 4/5] Update environment_cpu.yml Co-authored-by: Jacob Bieker --- environment_cpu.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment_cpu.yml b/environment_cpu.yml index 423814b6..db087fc2 100644 --- a/environment_cpu.yml +++ b/environment_cpu.yml @@ -10,7 +10,8 @@ dependencies: - pip - pyg - python=3.12 - - pytorch cpuonly -c pytorch + - pytorch + - cpuonly - pytorch-cluster - pytorch-scatter - pytorch-sparse From 4a5ce2a7ca225c8df381816e295fbe520b389ee2 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Tue, 2 Apr 2024 10:43:09 +0100 Subject: [PATCH 5/5] Update environment_cuda.yml --- environment_cuda.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/environment_cuda.yml b/environment_cuda.yml index 64ae707f..9b9de3b4 100644 --- a/environment_cuda.yml +++ b/environment_cuda.yml @@ -25,7 +25,6 @@ dependencies: - zarr - h3-py - numpy - - torch - pip: - datasets - einops