Merge branch 'main' into mvpatel2000/torch24

.github/PULL_REQUEST_TEMPLATE.md

@@ -16,13 +16,13 @@ Example:
 -->
 
 # Before submitting
-- [ ] Have you read the [contributor guidelines](https://github.com/mosaicml/composer/blob/dev/CONTRIBUTING.md)?
+- [ ] Have you read the [contributor guidelines](https://github.com/mosaicml/composer/blob/main/CONTRIBUTING.md)?
 - [ ] Is this change a documentation change or typo fix? If so, skip the rest of this checklist.
 - [ ] Was this change discussed/approved in a GitHub issue first? It is much more likely to be merged if so.
 - [ ] Did you update any related docs and document your change?
-- [ ] Did you update any related tests and add any new tests related to your change? (see [testing](https://github.com/mosaicml/composer/blob/dev/CONTRIBUTING.md#running-tests))
+- [ ] Did you update any related tests and add any new tests related to your change? (see [testing](https://github.com/mosaicml/composer/blob/main/CONTRIBUTING.md#running-tests))
 - [ ] Did you run the tests locally to make sure they pass?
-- [ ] Did you run `pre-commit` on your change? (see the `pre-commit` section of [prerequisites](https://github.com/mosaicml/composer/blob/dev/CONTRIBUTING.md#prerequisites))
+- [ ] Did you run `pre-commit` on your change? (see the `pre-commit` section of [prerequisites](https://github.com/mosaicml/composer/blob/main/CONTRIBUTING.md#prerequisites))
 
 <!--
 Thanks so much for contributing to composer! We really appreciate it :)

.github/workflows/code-quality.yaml

@@ -2,7 +2,6 @@ name: Code Quality Checks
 on:
   push:
     branches:
-    - dev
     - main
     - release/**
   pull_request:
@@ -19,6 +18,7 @@ jobs:
   code-quality:
     runs-on: ubuntu-20.04
     timeout-minutes: 15
+    if: github.repository_owner == 'mosaicml'
     strategy:
       matrix:
         python_version:

.github/workflows/daily.yaml

@@ -4,7 +4,6 @@ on:
   - cron: "30 2 * * *"  # 2:30 every day
   push:
     branches:
-    - dev
     - main
     - release/**
   workflow_dispatch:
@@ -77,13 +76,10 @@ jobs:
       pytest-command: ${{ matrix.pytest_command }}
       pytest-markers: ${{ matrix.markers }}
       composer_package_name: ${{ matrix.composer_package_name }}
-      pytest-wandb-entity: "mosaicml-public-integration-tests"
-      pytest-wandb-project: "integration-tests-${{ github.sha }}"
       safe_directory: composer
     secrets:
       aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      wandb-api-key: ${{ secrets.WANDB_API_KEY }}
       code-eval-device: ${{ secrets.CODE_EVAL_DEVICE }}
       code-eval-url: ${{ secrets.CODE_EVAL_URL }}
       code-eval-apikey: ${{ secrets.CODE_EVAL_APIKEY }}

.github/workflows/pr-docker.yaml

@@ -2,7 +2,6 @@ name: PR Docker
 on:
   pull_request:
     branches:
-    - dev
     - main
     - release/**
     paths:

.github/workflows/pr-gpu.yaml

@@ -6,7 +6,7 @@ on:
 # or dev
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 jobs:
   pytest-gpu-1:
     uses: mosaicml/ci-testing/.github/workflows/[email protected]

.github/workflows/smoketest.yaml

@@ -2,7 +2,6 @@ name: Smoketest
 on:
   push:
     branches:
-    - dev
     - main
     - release/**
   pull_request:
@@ -20,6 +19,7 @@ jobs:
   smoketest:
     runs-on: ubuntu-20.04
     timeout-minutes: 10
+    if: github.repository_owner == 'mosaicml'
     strategy:
       matrix:
         python_version:

composer/_version.py

@@ -3,4 +3,4 @@
 
 """The Composer Version."""
 
-__version__ = '0.23.5'
+__version__ = '0.24.0.dev0'

composer/checkpoint/__init__.py

@@ -10,11 +10,13 @@
     get_optim_state_dict,
     get_resumption_state_dict,
 )
+from composer.checkpoint.upload import upload_file
 
 __all__ = [
     'get_model_state_dict',
     'get_optim_state_dict',
     'get_metadata_state_dict',
     'get_resumption_state_dict',
     'download_monolithic_checkpoint',
+    'upload_file',
 ]

composer/checkpoint/download.py

@@ -40,10 +40,11 @@ def download_file(
     if node_ranks is not None and node_rank not in node_ranks:
         return
 
+    if source_uri.endswith('.symlink'):
+        source_path = download_and_extract_symlink(source_uri)
+
     object_store = maybe_create_object_store_from_uri(source_uri)
     _, _, source_path = parse_uri(source_uri)
-    if source_uri.endswith('.symlink'):
-        source_path = extract_path_from_symlink(source_path, object_store)
     assert object_store is not None
 
     @retry(num_attempts=num_attempts)
@@ -58,6 +59,21 @@ def _download():
     log.debug(f'Finished downloading {source_path} to {destination_path}')
 
 
+def download_and_extract_symlink(source_uri) -> str:
+    """Downloads a symlink file from the specified URI and returns the path it points to.
+
+    Args:
+        source_uri (str): The URI to download the symlink from.
+
+    Returns:
+        str: The path the symlink points to.
+    """
+    object_store = maybe_create_object_store_from_uri(source_uri)
+    _, _, source_path = parse_uri(source_uri)
+    source_path = extract_path_from_symlink(source_path, object_store)
+    return source_path
+
+
 def download_monolithic_checkpoint(
     source_uri: str,
     destination_path: str,