debug

mosaicml · Jan 19, 2024 · e491a2a · e491a2a
1 parent 1824dd3
commit e491a2a
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 130 deletions.
diff --git a/docker/README.md b/docker/README.md
@@ -32,20 +32,14 @@ To install composer, once inside the image, run `pip install mosaicml`.
 | Linux Distro   | Flavor   | PyTorch Version   | CUDA Version        | Python Version   | Docker Tags                                                                              |
 |----------------|----------|-------------------|---------------------|------------------|------------------------------------------------------------------------------------------|
 | Ubuntu 20.04   | Base     | 2.2.0             | 12.1.0 (Infiniband) | 3.10             | `mosaicml/pytorch:2.2.0_cu121-nightly20231213-python3.10-ubuntu20.04`                    |
-| Ubuntu 20.04   | Base     | 2.1.2             | 12.1.0 (Infiniband) | 3.11             | `mosaicml/pytorch:latest`, `mosaicml/pytorch:2.1.2_cu121-python3.11-ubuntu20.04`         |
-| Ubuntu 20.04   | Base     | 2.1.2             | 12.1.0 (EFA)        | 3.11             | `mosaicml/pytorch:latest-aws`, `mosaicml/pytorch:2.1.2_cu121-python3.11-ubuntu20.04-aws` |
-| Ubuntu 20.04   | Base     | 2.1.2             | cpu                 | 3.11             | `mosaicml/pytorch:latest_cpu`, `mosaicml/pytorch:2.1.2_cpu-python3.11-ubuntu20.04`       |
-| Ubuntu 20.04   | Base     | 2.1.2             | 12.1.0 (Infiniband) | 3.10             | `mosaicml/pytorch:2.1.2_cu121-python3.10-ubuntu20.04`                                    |
-| Ubuntu 20.04   | Base     | 2.1.2             | cpu                 | 3.10             | `mosaicml/pytorch:2.1.2_cpu-python3.10-ubuntu20.04`                                      |
-| Ubuntu 20.04   | Base     | 2.0.1             | 11.8.0 (Infiniband) | 3.11             | `mosaicml/pytorch:2.0.1_cu118-python3.11-ubuntu20.04`                                    |
-| Ubuntu 20.04   | Base     | 2.0.1             | 11.8.0 (EFA)        | 3.11             | `mosaicml/pytorch:2.0.1_cu118-python3.11-ubuntu20.04-aws`                                |
-| Ubuntu 20.04   | Base     | 2.0.1             | cpu                 | 3.11             | `mosaicml/pytorch:2.0.1_cpu-python3.11-ubuntu20.04`                                      |
+| Ubuntu 20.04   | Base     | 2.1.2             | 12.1.0 (Infiniband) | 3.10             | `mosaicml/pytorch:latest`, `mosaicml/pytorch:2.1.2_cu121-python3.10-ubuntu20.04`         |
+| Ubuntu 20.04   | Base     | 2.1.2             | 12.1.0 (EFA)        | 3.10             | `mosaicml/pytorch:latest-aws`, `mosaicml/pytorch:2.1.2_cu121-python3.10-ubuntu20.04-aws` |
+| Ubuntu 20.04   | Base     | 2.1.2             | cpu                 | 3.10             | `mosaicml/pytorch:latest_cpu`, `mosaicml/pytorch:2.1.2_cpu-python3.10-ubuntu20.04`       |
 | Ubuntu 20.04   | Base     | 2.0.1             | 11.8.0 (Infiniband) | 3.10             | `mosaicml/pytorch:2.0.1_cu118-python3.10-ubuntu20.04`                                    |
+| Ubuntu 20.04   | Base     | 2.0.1             | 11.8.0 (EFA)        | 3.10             | `mosaicml/pytorch:2.0.1_cu118-python3.10-ubuntu20.04-aws`                                |
 | Ubuntu 20.04   | Base     | 2.0.1             | cpu                 | 3.10             | `mosaicml/pytorch:2.0.1_cpu-python3.10-ubuntu20.04`                                      |
-| Ubuntu 20.04   | Base     | 1.13.1            | 11.7.1 (Infiniband) | 3.11             | `mosaicml/pytorch:1.13.1_cu117-python3.11-ubuntu20.04`                                   |
-| Ubuntu 20.04   | Base     | 1.13.1            | 11.7.1 (EFA)        | 3.11             | `mosaicml/pytorch:1.13.1_cu117-python3.11-ubuntu20.04-aws`                               |
-| Ubuntu 20.04   | Base     | 1.13.1            | cpu                 | 3.11             | `mosaicml/pytorch:1.13.1_cpu-python3.11-ubuntu20.04`                                     |
 | Ubuntu 20.04   | Base     | 1.13.1            | 11.7.1 (Infiniband) | 3.10             | `mosaicml/pytorch:1.13.1_cu117-python3.10-ubuntu20.04`                                   |
+| Ubuntu 20.04   | Base     | 1.13.1            | 11.7.1 (EFA)        | 3.10             | `mosaicml/pytorch:1.13.1_cu117-python3.10-ubuntu20.04-aws`                               |
 | Ubuntu 20.04   | Base     | 1.13.1            | cpu                 | 3.10             | `mosaicml/pytorch:1.13.1_cpu-python3.10-ubuntu20.04`                                     |
 <!-- END_PYTORCH_BUILD_MATRIX -->
 

diff --git a/docker/build_matrix.yaml b/docker/build_matrix.yaml
@@ -18,12 +18,12 @@
     brand=unknown,driver>=525,driver<526 brand=nvidia,driver>=525,driver<526 brand=nvidiartx,driver>=525,driver<526
     brand=geforce,driver>=525,driver<526 brand=geforcertx,driver>=525,driver<526 brand=quadro,driver>=525,driver<526
     brand=quadrortx,driver>=525,driver<526 brand=titan,driver>=525,driver<526 brand=titanrtx,driver>=525,driver<526
-  PYTHON_VERSION: '3.11'
+  PYTHON_VERSION: '3.10'
   PYTORCH_NIGHTLY_URL: ''
   PYTORCH_NIGHTLY_VERSION: ''
   PYTORCH_VERSION: 2.1.2
   TAGS:
-  - mosaicml/pytorch:2.1.2_cu121-python3.11-ubuntu20.04
+  - mosaicml/pytorch:2.1.2_cu121-python3.10-ubuntu20.04
   - mosaicml/pytorch:latest
   TARGET: pytorch_stage
   TORCHVISION_VERSION: 0.16.2
@@ -46,12 +46,12 @@
     brand=unknown,driver>=525,driver<526 brand=nvidia,driver>=525,driver<526 brand=nvidiartx,driver>=525,driver<526
     brand=geforce,driver>=525,driver<526 brand=geforcertx,driver>=525,driver<526 brand=quadro,driver>=525,driver<526
     brand=quadrortx,driver>=525,driver<526 brand=titan,driver>=525,driver<526 brand=titanrtx,driver>=525,driver<526
-  PYTHON_VERSION: '3.11'
+  PYTHON_VERSION: '3.10'
   PYTORCH_NIGHTLY_URL: ''
   PYTORCH_NIGHTLY_VERSION: ''
   PYTORCH_VERSION: 2.1.2
   TAGS:
-  - mosaicml/pytorch:2.1.2_cu121-python3.11-ubuntu20.04-aws
+  - mosaicml/pytorch:2.1.2_cu121-python3.10-ubuntu20.04-aws
   - mosaicml/pytorch:latest-aws
   TARGET: pytorch_stage
   TORCHVISION_VERSION: 0.16.2
@@ -61,12 +61,12 @@
   IMAGE_NAME: torch-2-1-2-cpu
   MOFED_VERSION: ''
   NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
-  PYTHON_VERSION: '3.11'
+  PYTHON_VERSION: '3.10'
   PYTORCH_NIGHTLY_URL: ''
   PYTORCH_NIGHTLY_VERSION: ''
   PYTORCH_VERSION: 2.1.2
   TAGS:
-  - mosaicml/pytorch:2.1.2_cpu-python3.11-ubuntu20.04
+  - mosaicml/pytorch:2.1.2_cpu-python3.10-ubuntu20.04
   - mosaicml/pytorch:latest_cpu
   TARGET: pytorch_stage
   TORCHVISION_VERSION: 0.16.2
@@ -81,12 +81,12 @@
     brand=geforce,driver>=470,driver<471 brand=geforce,driver>=515,driver<516 brand=quadro,driver>=470,driver<471
     brand=quadro,driver>=515,driver<516 brand=titan,driver>=470,driver<471 brand=titan,driver>=515,driver<516
     brand=titanrtx,driver>=470,driver<471 brand=titanrtx,driver>=515,driver<516
-  PYTHON_VERSION: '3.11'
+  PYTHON_VERSION: '3.10'
   PYTORCH_NIGHTLY_URL: ''
   PYTORCH_NIGHTLY_VERSION: ''
   PYTORCH_VERSION: 2.0.1
   TAGS:
-  - mosaicml/pytorch:2.0.1_cu118-python3.11-ubuntu20.04
+  - mosaicml/pytorch:2.0.1_cu118-python3.10-ubuntu20.04
   TARGET: pytorch_stage
   TORCHVISION_VERSION: 0.15.2
 - AWS_OFI_NCCL_VERSION: v1.7.4-aws
@@ -100,12 +100,12 @@
     brand=geforce,driver>=470,driver<471 brand=geforce,driver>=515,driver<516 brand=quadro,driver>=470,driver<471
     brand=quadro,driver>=515,driver<516 brand=titan,driver>=470,driver<471 brand=titan,driver>=515,driver<516
     brand=titanrtx,driver>=470,driver<471 brand=titanrtx,driver>=515,driver<516
-  PYTHON_VERSION: '3.11'
+  PYTHON_VERSION: '3.10'
   PYTORCH_NIGHTLY_URL: ''
   PYTORCH_NIGHTLY_VERSION: ''
   PYTORCH_VERSION: 2.0.1
   TAGS:
-  - mosaicml/pytorch:2.0.1_cu118-python3.11-ubuntu20.04-aws
+  - mosaicml/pytorch:2.0.1_cu118-python3.10-ubuntu20.04-aws
   TARGET: pytorch_stage
   TORCHVISION_VERSION: 0.15.2
 - AWS_OFI_NCCL_VERSION: ''
@@ -114,12 +114,12 @@
   IMAGE_NAME: torch-2-0-1-cpu
   MOFED_VERSION: ''
   NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
-  PYTHON_VERSION: '3.11'
+  PYTHON_VERSION: '3.10'
   PYTORCH_NIGHTLY_URL: ''
   PYTORCH_NIGHTLY_VERSION: ''
   PYTORCH_VERSION: 2.0.1
   TAGS:
-  - mosaicml/pytorch:2.0.1_cpu-python3.11-ubuntu20.04
+  - mosaicml/pytorch:2.0.1_cpu-python3.10-ubuntu20.04
   TARGET: pytorch_stage
   TORCHVISION_VERSION: 0.15.2
 - AWS_OFI_NCCL_VERSION: ''
@@ -128,12 +128,12 @@
   IMAGE_NAME: torch-1-13-1-cu117
   MOFED_VERSION: 5.5-1.0.3.2
   NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
-  PYTHON_VERSION: '3.11'
+  PYTHON_VERSION: '3.10'
   PYTORCH_NIGHTLY_URL: ''
   PYTORCH_NIGHTLY_VERSION: ''
   PYTORCH_VERSION: 1.13.1
   TAGS:
-  - mosaicml/pytorch:1.13.1_cu117-python3.11-ubuntu20.04
+  - mosaicml/pytorch:1.13.1_cu117-python3.10-ubuntu20.04
   TARGET: pytorch_stage
   TORCHVISION_VERSION: 0.14.1
 - AWS_OFI_NCCL_VERSION: v1.7.4-aws
@@ -142,114 +142,12 @@
   IMAGE_NAME: torch-1-13-1-cu117-aws
   MOFED_VERSION: ''
   NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
-  PYTHON_VERSION: '3.11'
-  PYTORCH_NIGHTLY_URL: ''
-  PYTORCH_NIGHTLY_VERSION: ''
-  PYTORCH_VERSION: 1.13.1
-  TAGS:
-  - mosaicml/pytorch:1.13.1_cu117-python3.11-ubuntu20.04-aws
-  TARGET: pytorch_stage
-  TORCHVISION_VERSION: 0.14.1
-- AWS_OFI_NCCL_VERSION: ''
-  BASE_IMAGE: ubuntu:20.04
-  CUDA_VERSION: ''
-  IMAGE_NAME: torch-1-13-1-cpu
-  MOFED_VERSION: ''
-  NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
-  PYTHON_VERSION: '3.11'
-  PYTORCH_NIGHTLY_URL: ''
-  PYTORCH_NIGHTLY_VERSION: ''
-  PYTORCH_VERSION: 1.13.1
-  TAGS:
-  - mosaicml/pytorch:1.13.1_cpu-python3.11-ubuntu20.04
-  TARGET: pytorch_stage
-  TORCHVISION_VERSION: 0.14.1
-- AWS_OFI_NCCL_VERSION: ''
-  BASE_IMAGE: nvidia/cuda:12.1.0-cudnn8-devel-ubuntu20.04
-  CUDA_VERSION: 12.1.0
-  IMAGE_NAME: torch-2-1-2-cu121
-  MOFED_VERSION: 5.5-1.0.3.2
-  NVIDIA_REQUIRE_CUDA_OVERRIDE: cuda>=12.1 brand=tesla,driver>=450,driver<451 brand=tesla,driver>=470,driver<471
-    brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471
-    brand=geforce,driver>=470,driver<471 brand=geforcertx,driver>=470,driver<471 brand=quadro,driver>=470,driver<471
-    brand=quadrortx,driver>=470,driver<471 brand=titan,driver>=470,driver<471 brand=titanrtx,driver>=470,driver<471
-    brand=tesla,driver>=510,driver<511 brand=unknown,driver>=510,driver<511 brand=nvidia,driver>=510,driver<511
-    brand=nvidiartx,driver>=510,driver<511 brand=geforce,driver>=510,driver<511 brand=geforcertx,driver>=510,driver<511
-    brand=quadro,driver>=510,driver<511 brand=quadrortx,driver>=510,driver<511 brand=titan,driver>=510,driver<511
-    brand=titanrtx,driver>=510,driver<511 brand=tesla,driver>=515,driver<516 brand=unknown,driver>=515,driver<516
-    brand=nvidia,driver>=515,driver<516 brand=nvidiartx,driver>=515,driver<516 brand=geforce,driver>=515,driver<516
-    brand=geforcertx,driver>=515,driver<516 brand=quadro,driver>=515,driver<516 brand=quadrortx,driver>=515,driver<516
-    brand=titan,driver>=515,driver<516 brand=titanrtx,driver>=515,driver<516 brand=tesla,driver>=525,driver<526
-    brand=unknown,driver>=525,driver<526 brand=nvidia,driver>=525,driver<526 brand=nvidiartx,driver>=525,driver<526
-    brand=geforce,driver>=525,driver<526 brand=geforcertx,driver>=525,driver<526 brand=quadro,driver>=525,driver<526
-    brand=quadrortx,driver>=525,driver<526 brand=titan,driver>=525,driver<526 brand=titanrtx,driver>=525,driver<526
-  PYTHON_VERSION: '3.10'
-  PYTORCH_NIGHTLY_URL: ''
-  PYTORCH_NIGHTLY_VERSION: ''
-  PYTORCH_VERSION: 2.1.2
-  TAGS:
-  - mosaicml/pytorch:2.1.2_cu121-python3.10-ubuntu20.04
-  TARGET: pytorch_stage
-  TORCHVISION_VERSION: 0.16.2
-- AWS_OFI_NCCL_VERSION: ''
-  BASE_IMAGE: ubuntu:20.04
-  CUDA_VERSION: ''
-  IMAGE_NAME: torch-2-1-2-cpu
-  MOFED_VERSION: ''
-  NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
-  PYTHON_VERSION: '3.10'
-  PYTORCH_NIGHTLY_URL: ''
-  PYTORCH_NIGHTLY_VERSION: ''
-  PYTORCH_VERSION: 2.1.2
-  TAGS:
-  - mosaicml/pytorch:2.1.2_cpu-python3.10-ubuntu20.04
-  TARGET: pytorch_stage
-  TORCHVISION_VERSION: 0.16.2
-- AWS_OFI_NCCL_VERSION: ''
-  BASE_IMAGE: nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04
-  CUDA_VERSION: 11.8.0
-  IMAGE_NAME: torch-2-0-1-cu118
-  MOFED_VERSION: 5.5-1.0.3.2
-  NVIDIA_REQUIRE_CUDA_OVERRIDE: cuda>=11.8 brand=tesla,driver>=470,driver<471 brand=tesla,driver>=515,driver<516
-    brand=unknown,driver>=470,driver<471 brand=unknown,driver>=515,driver<516 brand=nvidia,driver>=470,driver<471
-    brand=nvidia,driver>=515,driver<516 brand=nvidiartx,driver>=470,driver<471 brand=nvidiartx,driver>=515,driver<516
-    brand=geforce,driver>=470,driver<471 brand=geforce,driver>=515,driver<516 brand=quadro,driver>=470,driver<471
-    brand=quadro,driver>=515,driver<516 brand=titan,driver>=470,driver<471 brand=titan,driver>=515,driver<516
-    brand=titanrtx,driver>=470,driver<471 brand=titanrtx,driver>=515,driver<516
-  PYTHON_VERSION: '3.10'
-  PYTORCH_NIGHTLY_URL: ''
-  PYTORCH_NIGHTLY_VERSION: ''
-  PYTORCH_VERSION: 2.0.1
-  TAGS:
-  - mosaicml/pytorch:2.0.1_cu118-python3.10-ubuntu20.04
-  TARGET: pytorch_stage
-  TORCHVISION_VERSION: 0.15.2
-- AWS_OFI_NCCL_VERSION: ''
-  BASE_IMAGE: ubuntu:20.04
-  CUDA_VERSION: ''
-  IMAGE_NAME: torch-2-0-1-cpu
-  MOFED_VERSION: ''
-  NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
-  PYTHON_VERSION: '3.10'
-  PYTORCH_NIGHTLY_URL: ''
-  PYTORCH_NIGHTLY_VERSION: ''
-  PYTORCH_VERSION: 2.0.1
-  TAGS:
-  - mosaicml/pytorch:2.0.1_cpu-python3.10-ubuntu20.04
-  TARGET: pytorch_stage
-  TORCHVISION_VERSION: 0.15.2
-- AWS_OFI_NCCL_VERSION: ''
-  BASE_IMAGE: nvidia/cuda:11.7.1-cudnn8-devel-ubuntu20.04
-  CUDA_VERSION: 11.7.1
-  IMAGE_NAME: torch-1-13-1-cu117
-  MOFED_VERSION: 5.5-1.0.3.2
-  NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
   PYTHON_VERSION: '3.10'
   PYTORCH_NIGHTLY_URL: ''
   PYTORCH_NIGHTLY_VERSION: ''
   PYTORCH_VERSION: 1.13.1
   TAGS:
-  - mosaicml/pytorch:1.13.1_cu117-python3.10-ubuntu20.04
+  - mosaicml/pytorch:1.13.1_cu117-python3.10-ubuntu20.04-aws
   TARGET: pytorch_stage
   TORCHVISION_VERSION: 0.14.1
 - AWS_OFI_NCCL_VERSION: ''
@@ -313,7 +211,7 @@
     brand=unknown,driver>=525,driver<526 brand=nvidia,driver>=525,driver<526 brand=nvidiartx,driver>=525,driver<526
     brand=geforce,driver>=525,driver<526 brand=geforcertx,driver>=525,driver<526 brand=quadro,driver>=525,driver<526
     brand=quadrortx,driver>=525,driver<526 brand=titan,driver>=525,driver<526 brand=titanrtx,driver>=525,driver<526
-  PYTHON_VERSION: '3.11'
+  PYTHON_VERSION: '3.10'
   PYTORCH_NIGHTLY_URL: ''
   PYTORCH_NIGHTLY_VERSION: ''
   PYTORCH_VERSION: 2.1.2
@@ -329,7 +227,7 @@
   IMAGE_NAME: composer-0-17-2-cpu
   MOFED_VERSION: 5.5-1.0.3.2
   NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
-  PYTHON_VERSION: '3.11'
+  PYTHON_VERSION: '3.10'
   PYTORCH_NIGHTLY_URL: ''
   PYTORCH_NIGHTLY_VERSION: ''
   PYTORCH_VERSION: 2.1.2