Cleanup dependencies (#2515)

* cleaup some dependencies * Remove upper bound for click * upgrade actions * upgrade requirements docs * Add test for poetry --lock * fix poetry test * rename poetry test * fix * fix path * fix github env var * fix * fix poetry * Adopt changes from #2514 * increase poetry build time
skypilot-org · Sep 8, 2023 · 6f1d4e8 · 6f1d4e8
1 parent eb8599d
commit 6f1d4e8
Show file tree

Hide file tree

Showing 10 changed files with 69 additions and 26 deletions.
diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml
@@ -18,9 +18,9 @@ jobs:
       matrix:
         python-version: ["3.8"]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml
@@ -18,9 +18,9 @@ jobs:
       matrix:
         python-version: ["3.8"]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml
@@ -19,9 +19,9 @@ jobs:
       matrix:
         python-version: ["3.8"]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

diff --git a/.github/workflows/pypi-nightly-build.yml b/.github/workflows/pypi-nightly-build.yml
@@ -11,7 +11,7 @@ jobs:
     outputs:
       should_run: ${{ steps.should_run.outputs.should_run }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: print latest_commit
         run: echo ${{ github.sha }}
       - id: should_run

diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -24,17 +24,16 @@ jobs:
           - tests/test_list_accelerators.py
           - tests/test_optimizer_dryruns.py
           - tests/test_optimizer_random_dag.py
-          - tests/test_pycryptodome_version.py
           - tests/test_storage.py
           - tests/test_wheels.py
           - tests/test_spot.py
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Install Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/test-poetry-build.yml b/.github/workflows/test-poetry-build.yml
@@ -0,0 +1,53 @@
+name: Poetry Test
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  push:
+    branches:
+      - master
+      - 'releases/**'
+  pull_request:
+    branches:
+      - master
+      - 'releases/**'
+jobs:
+  poetry-build-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python -
+          echo "$HOME/.poetry/bin" >> $GITHUB_PATH
+      - name: Create foo package
+        run: |
+          mkdir foo
+          cat <<EOF > foo/pyproject.toml
+          [tool.poetry]
+          name = "foo"
+          version = "1.0.0"
+          authors = ["skypilot-bot"]
+          description = ""
+
+          [tool.poetry.dependencies]
+          python = "3.10.x"
+
+          [tool.poetry.group.dev.dependencies]
+          skypilot = {git = "https://github.com/skypilot-org/skypilot.git", branch = "${{ github.head_ref }}"}
+
+          [build-system]
+          requires = ["poetry-core"]
+          build-backend = "poetry.core.masonry.api"
+          
+          EOF
+
+      - name: Check poetry lock time
+        run: |
+          cd foo
+          poetry lock --no-update
+        timeout-minutes: 2
+
+
diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt
@@ -1,5 +1,5 @@
 sphinx==4.3.2
-sphinx-click==3.0.2
+sphinx-click==5.0.1
 sphinx-copybutton==0.5.0
 pydata-sphinx-theme==0.7.2
 sphinx-autodoc-typehints==1.17.0

diff --git a/sky/data/data_transfer.py b/sky/data/data_transfer.py
@@ -47,12 +47,12 @@ def s3_to_gcs(s3_bucket_name: str, gs_bucket_name: str) -> None:
       gs_bucket_name: str; Name of the Google Cloud Storage Bucket
     """
     # pylint: disable=import-outside-toplevel
-    from oauth2client.client import GoogleCredentials
+    import google.auth
 
-    oauth_credentials = GoogleCredentials.get_application_default()
+    credentials, _ = google.auth.default()
     storagetransfer = gcp.build('storagetransfer',
                                 'v1',
-                                credentials=oauth_credentials)
+                                credentials=credentials)
 
     session = aws.session()
     aws_credentials = session.get_credentials().get_frozen_credentials()

diff --git a/sky/setup_files/setup.py b/sky/setup_files/setup.py
@@ -56,11 +56,8 @@ def parse_readme(readme: str) -> str:
 
 install_requires = [
     'wheel',
-    # NOTE: ray requires click>=7.0. Also, click 8.1.x makes our rendered CLI
-    # docs display weird blockquotes.
-    # TODO(zongheng): investigate how to make click 8.1.x display nicely and
-    # remove the upper bound.
-    'click <= 8.0.4, >= 7.0',
+    # NOTE: ray requires click>=7.0.
+    'click >= 7.0',
     # NOTE: required by awscli. To avoid ray automatically installing
     # the latest version.
     'colorama < 0.4.5',
@@ -71,7 +68,6 @@ def parse_readme(readme: str) -> str:
     'jinja2 >= 3.0',
     'jsonschema',
     'networkx',
-    'oauth2client',
     'pandas',
     'pendulum',
     # PrettyTable with version >=2.0.0 is required for the support of
@@ -117,7 +113,7 @@ def parse_readme(readme: str) -> str:
 # NOTE: Change the templates/spot-controller.yaml.j2 file if any of the
 # following packages dependencies are changed.
 aws_dependencies = [
-    # botocore does not work with urllib3>=2.0.0, accuroding to https://github.com/boto/botocore/issues/2926
+    # botocore does not work with urllib3>=2.0.0, according to https://github.com/boto/botocore/issues/2926
     # We have to explicitly pin the version to optimize the time for
     # poetry install. See https://github.com/orgs/python-poetry/discussions/7937
     'urllib3<2',
@@ -127,8 +123,6 @@ def parse_readme(readme: str) -> str:
     'awscli>=1.27.10',
     'botocore>=1.29.10',
     'boto3>=1.26.1',
-    # 'Crypto' module used in authentication.py for AWS.
-    'pycryptodome==3.12.0',
 ]
 extras_require: Dict[str, List[str]] = {
     'aws': aws_dependencies,

diff --git a/tests/test_pycryptodome_version.py b/tests/test_pycryptodome_version.py