Merge pull request Backblaze#991 from Backblaze/pdm

Manage dependencies with PDM
reef-technologies · Feb 5, 2024 · 0693535 · 0693535
2 parents 92ef2c8 + 3463fff
commit 0693535
Show file tree

Hide file tree

Showing 20 changed files with 1,966 additions and 373 deletions.
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -25,7 +25,7 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
       - name: Install dependencies
-        run: python -m pip install --upgrade nox pip setuptools
+        run: python -m pip install --upgrade nox pdm
       - name: Build the distribution
         id: build
         run: nox -vs build >> $GITHUB_OUTPUT
@@ -64,7 +64,7 @@ jobs:
         run: |
           apt-get -y update
           apt-get -y install patchelf
-          python -m pip install --upgrade nox pip setuptools
+          python -m pip install --upgrade nox pdm
           git config --global --add safe.directory '*'
       - name: Bundle the distribution
         id: bundle
@@ -97,7 +97,7 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
       - name: Install dependencies
-        run: python -m pip install --upgrade nox pip setuptools
+        run: python -m pip install --upgrade nox pdm
       - name: Bundle the distribution
         id: bundle
         shell: bash

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -20,11 +20,11 @@ jobs:
         with:
           ignore_words_list: datas re-use
       - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
       - name: Install dependencies
-        run: python -m pip install --upgrade nox pip setuptools
+        run: python -m pip install --upgrade nox pdm
       - name: Run linters
         run: nox -vs lint
       - name: Validate new changelog entries
@@ -45,7 +45,7 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
       - name: Install dependencies
-        run: python -m pip install --upgrade nox pip setuptools
+        run: python -m pip install --upgrade nox pdm
       - name: Build the distribution
         run: nox -vs build >> $GITHUB_OUTPUT
   cleanup_buckets:
@@ -67,7 +67,7 @@ jobs:
           cache: "pip"
       - name: Install dependencies
         if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}  # TODO: skip this whole job instead
-        run: python -m pip install --upgrade nox pip setuptools
+        run: python -m pip install --upgrade nox pdm
       - name: Find and remove old buckets
         if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}  # TODO: skip this whole job instead
         run: nox -vs cleanup_buckets
@@ -81,31 +81,30 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        # pypy version pin was required due 7.3.13 being broken https://foss.heptapod.net/pypy/pypy/-/issues/4021
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.10-nightly"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy3.9", "pypy3.10"]
         exclude:
           - os: "macos-latest"
-            python-version: "pypy-3.10-nightly"
+            python-version: "pypy3.10"
           - os: "windows-latest"
-            python-version: "pypy-3.10-nightly"
+            python-version: "pypy3.10"
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
-        run: python -m pip install --upgrade nox pip setuptools
+        run: python -m pip install --upgrade nox pdm
       - name: Run unit tests
-        run: nox -vs unit
+        run: nox -vs unit -p ${{ matrix.python-version }}
       - name: Run integration tests (without secrets)
-        run: nox -vs integration -- -m "not require_secrets"
+        run: nox -vs integration -p ${{ matrix.python-version }} -- -m "not require_secrets"
       - name: Run integration tests (with secrets)
         # Limit CI workload by running integration tests with secrets only on edge Python versions.
-        if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' && contains(fromJSON('["3.7", "pypy-3.10-nightly", "3.12"]'), matrix.python-version) }}
-        run: nox -vs integration -- -m "require_secrets" --cleanup
+        if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' && contains(fromJSON('["3.7", "pypy3.10", "3.12"]'), matrix.python-version) }}
+        run: nox -vs integration -p ${{ matrix.python-version }} -- -m "require_secrets" --cleanup
   test-docker:
     needs: cleanup_buckets
     env:
@@ -117,13 +116,11 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
-      - name: setup sudo NOX_PYTHONS
-        run: echo NOX_PYTHONS=$(sudo python3 --version | cut -d ' ' -f 2) >> "$GITHUB_ENV"
       - name: Install dependencies
-        run: sudo python -m pip install --upgrade nox pip setuptools
+        run: sudo python -m pip install --upgrade nox pdm
       - name: Generate Dockerfile
         run: nox -vs generate_dockerfile
       - name: Set up QEMU
@@ -139,7 +136,7 @@ jobs:
           platforms: linux/amd64
       - name: Run tests with docker
         if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}
-        run: sudo NOX_PYTHONS=$NOX_PYTHONS B2_TEST_APPLICATION_KEY=${{ env.B2_TEST_APPLICATION_KEY }} B2_TEST_APPLICATION_KEY_ID=${{ env.B2_TEST_APPLICATION_KEY_ID }} nox -vs docker_test -- backblazeit/b2:test
+        run: nox -vs docker_test -- backblazeit/b2:test
   test-linux-bundle:
     needs: cleanup_buckets
     env:
@@ -158,7 +155,7 @@ jobs:
         run: |
           apt-get -y update
           apt-get -y install patchelf
-          python -m pip install --upgrade nox pip setuptools
+          python -m pip install --upgrade nox pdm
           git config --global --add safe.directory '*'
       - name: Bundle the distribution
         id: bundle
@@ -167,10 +164,10 @@ jobs:
         id: hashes
         run: nox -vs make_dist_digest
       - name: Run integration tests (without secrets)
-        run: nox -vs integration -- --sut=${{ steps.bundle.outputs.sut_path }} -m "not require_secrets"
+        run: nox -vs integration -p ${{ env.PYTHON_DEFAULT_VERSION }} -- --sut=${{ steps.bundle.outputs.sut_path }} -m "not require_secrets"
       - name: Run integration tests (with secrets)
         if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}
-        run: nox -vs integration -- --sut=${{ steps.bundle.outputs.sut_path }} -m "require_secrets" --cleanup
+        run: nox -vs integration -p ${{ env.PYTHON_DEFAULT_VERSION }} -- --sut=${{ steps.bundle.outputs.sut_path }} -m "require_secrets" --cleanup
       - name: Upload assets
         if: failure()
         uses: actions/upload-artifact@v2
@@ -193,11 +190,11 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
       - name: Install dependencies
-        run: python -m pip install --upgrade nox pip setuptools
+        run: python -m pip install --upgrade nox pdm
       - name: Bundle the distribution
         id: bundle
         shell: bash
@@ -206,10 +203,10 @@ jobs:
         id: hashes
         run: nox -vs make_dist_digest
       - name: Run integration tests (without secrets)
-        run: nox -vs integration -- --sut=${{ steps.bundle.outputs.sut_path }} -m "not require_secrets"
+        run: nox -vs integration -p ${{ env.PYTHON_DEFAULT_VERSION }} -- --sut=${{ steps.bundle.outputs.sut_path }} -m "not require_secrets"
       - name: Run integration tests (with secrets)
         if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}
-        run: nox -vs integration -- --sut=${{ steps.bundle.outputs.sut_path }} -m "require_secrets" --cleanup
+        run: nox -vs integration -p ${{ env.PYTHON_DEFAULT_VERSION }} -- --sut=${{ steps.bundle.outputs.sut_path }} -m "require_secrets" --cleanup
       - name: Upload assets
         if: failure()
         uses: actions/upload-artifact@v2
@@ -225,7 +222,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
       - name: Install dependencies
@@ -234,6 +231,6 @@ jobs:
         run: |
           sudo apt-get update -y
           sudo apt-get install -y graphviz plantuml
-          python -m pip install --upgrade nox pip setuptools
+          python -m pip install --upgrade nox pdm
       - name: Build the docs
         run: nox --non-interactive -vs doc
diff --git a/.github/workflows/push_docker.yml b/.github/workflows/push_docker.yml
@@ -21,7 +21,7 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
       - name: Install dependencies
-        run: python -m pip install --upgrade nox pip setuptools
+        run: python -m pip install --upgrade nox pdm
       - name: Build Dockerfile
         run: nox -vs generate_dockerfile
       - name: Set up QEMU
@@ -34,11 +34,9 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: install setuptools_scm
-        run: pip install setuptools_scm
       - name: get version
         id: package_version
-        run: echo package_version=`python -m setuptools_scm` >> $GITHUB_OUTPUT
+        run: echo package_version=`pdm show --version --quiet` >> $GITHUB_OUTPUT
       - name: echo
         run: echo ${{ steps.package_version.outputs.package_version }}
       - name: Build and push

diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,8 @@
 .eggs/
 .idea
 .nox/
+.pdm-build/
+.pdm-python
 .python-version
 b2_cli.log
 b2.egg-info
@@ -14,4 +16,4 @@ venv
 doc/source/main_help.rst
 Dockerfile
 b2/licenses_output.txt
-*.spec
+*.spec
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -8,7 +8,11 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.11"
+    python: "3.12"
+  jobs:
+    post_create_environment:
+      - pip install pdm
+      - pdm export --format requirements --prod --group doc --output requirements-doc.txt --no-hashes
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
@@ -20,8 +24,6 @@ formats: all
 # Optionally set the version of Python and requirements required to build your docs
 python:
   install:
-    - requirements: requirements.txt
+    - requirements: requirements-doc.txt
     - method: pip
-      path: .
-      extra_requirements:
-      - doc
+      path: .
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -55,9 +55,9 @@ for a given github issue).
 
 ## Developer Info
 
-You'll need to have [nox](https://github.com/theacodes/nox) installed:
+You'll need to have [nox](https://github.com/theacodes/nox) and [pdm](https://pdm-project.org/) installed:
 
-* `pip install nox`
+* `pip install nox pdm`
 
 With `nox`, you can run different sessions (default are `lint` and `test`):
 
@@ -69,7 +69,6 @@ With `nox`, you can run different sessions (default are `lint` and `test`):
 * `generate_dockerfile` -> generate dockerfile
 * `docker_test` -> run integration tests against a docker image
 * `build_and_test_docker` -> build a docker image and integration tests against it
-* `deploy` -> Deploy the distribution to the PyPi.
 * `doc` -> Build the documentation.
 * `doc_cover` -> Perform coverage analysis for the documentation.
 
@@ -105,6 +104,14 @@ With the above setting, session `test` will run on Python 3.9 and 3.10, and all
 
 Given Python interpreters should be installed in the operating system or via [pyenv](https://github.com/pyenv/pyenv).
 
+## Managing dependencies
+
+We use [pdm](https://pdm-project.org/) for managing dependencies and developing locally.
+If you want to change any of the project requirements (or requirement bounds) in `pyproject.toml`,
+make sure that `pdm.lock` file reflects those changes by using `pdm add`, `pdm update` or other
+commands - see [documentation](https://pdm-project.org/latest/). You can verify that lock file
+is up to date by running the linter.
+
 ## Linting
 
 To run all available linters:

diff --git a/Dockerfile.template b/Dockerfile.template
@@ -1,4 +1,18 @@
-FROM python:${python_version}-slim as base
+FROM python:${python_version}-slim as builder
+
+RUN apt-get update -y && apt-get install git patchelf -y && pip install -U pdm
+
+WORKDIR /b2
+COPY ./b2 /b2/b2
+COPY pyproject.toml pdm.lock LICENSE README.md /b2/
+
+ENV PDM_BUILD_SCM_VERSION=${version}
+RUN pdm install --prod --group license
+RUN pdm run b2 license --dump --with-packages
+# Run pdm in PEP 582 mode, install packaged to __pypackages__, not virtualenv
+RUN rm -r .venv && mkdir __pypackages__ && pdm install --prod --group full --no-editable
+
+FROM python:${python_version}-slim
 
 LABEL vendor=${vendor}
 LABEL name="${name}"
@@ -9,11 +23,10 @@ LABEL vcs-url="${vcs_url}"
 LABEL vcs-ref="${vcs_ref}"
 LABEL build-date-iso8601="${build_date}"
 
-WORKDIR /root
-
-# due https://github.com/moby/moby/issues/47021 we cannot have /root/.cache leftover as it causes random errors in CI
-RUN --mount=type=bind,source=${tar_path}/${tar_name},target=/tmp/${tar_name} \
-  pip install --no-cache-dir /tmp/${tar_name}[full] && rm -rf /root/.cache
+ENV PYTHONPATH=/opt/b2
+COPY --from=builder /b2/__pypackages__/${python_version}/lib /opt/b2
+COPY --from=builder /b2/__pypackages__/${python_version}/bin/* /bin/
 
+WORKDIR /root
 ENTRYPOINT ["b2"]
 CMD ["--help"]
diff --git a/MANIFEST.in b/MANIFEST.in
diff --git a/changelog.d/+pdm.infrastructure.md b/changelog.d/+pdm.infrastructure.md
@@ -0,0 +1 @@
+Use pdm for building, testing and managing dependencies. Remove unnecessary files (continuous integration scripts, tests) from sdist tarball.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Use pdm for building, testing and managing dependencies. Remove unnecessary files (continuous integration scripts, tests) from sdist tarball.