From e05c1b20752de87d0ee8eb6c20855b9040a077b3 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 4 Dec 2024 02:08:51 +0100 Subject: [PATCH] CI: avoid annoying warnings of OSSF ScoreCard about unpinned pip install --- .github/workflows/alpine/Dockerfile.ci | 2 +- .github/workflows/alpine_32bit/Dockerfile.ci | 2 +- .github/workflows/cmake_builds.yml | 14 +++++++------- .github/workflows/code_checks.yml | 6 +++--- .github/workflows/codeql.yml | 2 +- .github/workflows/doc_checks.yml | 10 +++++----- .github/workflows/fedora_rawhide/Dockerfile.ci | 4 ++-- .github/workflows/icc/Dockerfile.ci | 2 +- .github/workflows/ubuntu_20.04/Dockerfile.ci | 6 +++--- .github/workflows/ubuntu_22.04/Dockerfile.ci | 2 +- .github/workflows/ubuntu_24.04/Dockerfile.ci | 6 +++--- ci/travis/graviton2/script.sh | 3 ++- ci/travis/s390x/script.sh | 3 ++- scripts/vagrant/gdal.sh | 2 +- swig/python/gdal-utils/test-ubuntu-install.sh.txt | 6 +++--- 15 files changed, 36 insertions(+), 34 deletions(-) diff --git a/.github/workflows/alpine/Dockerfile.ci b/.github/workflows/alpine/Dockerfile.ci index 05d1c09c09d7..0a41699e3181 100644 --- a/.github/workflows/alpine/Dockerfile.ci +++ b/.github/workflows/alpine/Dockerfile.ci @@ -72,7 +72,7 @@ RUN apk add \ zstd-dev COPY requirements.txt /tmp/ -RUN python3 -m pip install --break-system-packages -U -r /tmp/requirements.txt +RUN PYTHON_CMD=python3 && $PYTHON_CMD -m pip install --break-system-packages -U -r /tmp/requirements.txt RUN apk add git autoconf automake libtool bison && \ git clone --branch "${SWIG_GIT_TAG:-master}" --depth 1 https://github.com/swig/swig.git swig-git && \ diff --git a/.github/workflows/alpine_32bit/Dockerfile.ci b/.github/workflows/alpine_32bit/Dockerfile.ci index 7080b05992bb..e5eac11f1cb0 100644 --- a/.github/workflows/alpine_32bit/Dockerfile.ci +++ b/.github/workflows/alpine_32bit/Dockerfile.ci @@ -74,4 +74,4 @@ RUN apk add \ zstd-dev COPY requirements.txt /tmp/ -RUN python3 -m pip install --break-system-packages -U -r /tmp/requirements.txt +RUN PYTHON_CMD=python3 && $PYTHON_CMD -m pip install --break-system-packages -U -r /tmp/requirements.txt diff --git a/.github/workflows/cmake_builds.yml b/.github/workflows/cmake_builds.yml index e66323d594e7..f4935a41221a 100644 --- a/.github/workflows/cmake_builds.yml +++ b/.github/workflows/cmake_builds.yml @@ -97,8 +97,8 @@ jobs: # Workaround bug in ogdi packaging sudo ln -s /usr/lib/ogdi/libvrf.so /usr/lib # - python3 -m pip install -U pip wheel setuptools numpy importlib_metadata - python3 -m pip install -r $GITHUB_WORKSPACE/autotest/requirements.txt + PYTHON_CMD=python3 && $PYTHON_CMD -m pip install -U pip wheel setuptools numpy importlib_metadata + PYTHON_CMD=python3 && $PYTHON_CMD -m pip install -r $GITHUB_WORKSPACE/autotest/requirements.txt - name: Build libjxl run: | @@ -245,7 +245,7 @@ jobs: python3 -m venv myvenv source myvenv/bin/activate # Set PATH so that gdal-config is found - PATH=$GITHUB_WORKSPACE/install-gdal/bin:$PATH pip install gdal-python.tar.gz[numpy] + PYTHON_CMD=python3 && PATH=$GITHUB_WORKSPACE/install-gdal/bin:$PATH $PYTHON_CMD -m pip install gdal-python.tar.gz[numpy] LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install-gdal/lib python -c "from osgeo import gdal_array" which gdal_edit cp $GITHUB_WORKSPACE/autotest/gcore/data/byte.tif . @@ -262,7 +262,7 @@ jobs: python3 -m venv myvenv source myvenv/bin/activate # Set PATH so that gdal-config is found - PATH=$GITHUB_WORKSPACE/install-gdal/bin:$PATH pip install gdal_utils-X.Y.Z.T-py3-none-any.whl + PYTHON_CMD=python3 && PATH=$GITHUB_WORKSPACE/install-gdal/bin:$PATH $PYTHON_CMD -m pip install gdal_utils-X.Y.Z.T-py3-none-any.whl which gdal_edit cp $GITHUB_WORKSPACE/autotest/gcore/data/byte.tif . LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install-gdal/lib gdal_edit byte.tif -mo FOO=BAR @@ -353,8 +353,8 @@ jobs: - name: Install python dependencies for autotest run: | # One of the dependencies of jsonschema 4.18 is rpds_py which requires a Rust compiler - python -m pip install "jsonschema<4.18" - python -m pip install -r autotest/requirements.txt + PYTHON_CMD=python3 && $PYTHON_CMD -m pip install "jsonschema<4.18" + PYTHON_CMD=python3 && $PYTHON_CMD -m pip install -r autotest/requirements.txt # Disable mySQL since C:/mysql/lib/mysqlclient.lib (unrelated to msys) is found, which causes linking issues # Set explicitly CMAKE_C|CXX_COMPILER otherwise C:/ProgramData/chocolatey/bin/gcc.exe would be used # Disable GDAL_ENABLE_DRIVER_HDF5 because of https://github.com/OSGeo/gdal/issues/11181 @@ -653,7 +653,7 @@ jobs: export PKG_CONFIG_PATH=/opt/homebrew/Cellar/libxml2/lib/pkgconfig # FIXME: remove -DBUILD_PYTHON_BINDINGS=OFF. Python tests fail with "ModuleNotFoundError: No module named '_gdal'" with macos-12 cmake -Werror=dev -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install-gdal ${CMAKE_OPTIONS} -DCMAKE_C_FLAGS=-Werror -DCMAKE_CXX_FLAGS=-Werror -DUSE_CCACHE=ON -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DWERROR_DEV_FLAG="-Werror=dev" -DBUILD_PYTHON_BINDINGS=OFF "-DUSE_PRECOMPILED_HEADERS=ON" - #$(cat $GITHUB_WORKSPACE/build/python_executable.txt) -m pip install --user -r $GITHUB_WORKSPACE/autotest/requirements.txt + #PYTHON_CMD=$(cat $GITHUB_WORKSPACE/build/python_executable.txt) && $PYTHON_CMD -m pip install --user -r $GITHUB_WORKSPACE/autotest/requirements.txt - name: Build run: | cmake --build $GITHUB_WORKSPACE/build -j 3 diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml index 629a7d81d6dc..88e8d9cf5622 100644 --- a/.github/workflows/code_checks.yml +++ b/.github/workflows/code_checks.yml @@ -178,7 +178,7 @@ jobs: run: | sudo apt install python3-pip wget # ruamel.yaml.clib 0.2.9 throws a 'TypeError: a string or stream input is required' when running cffconvert --validate - sudo pip3 install cffconvert "ruamel.yaml.clib<0.2.9" + PYTHON_CMD=python3 && sudo $PYTHON_CMD -m pip install cffconvert "ruamel.yaml.clib<0.2.9" - name: Validate citation file run: | @@ -199,8 +199,8 @@ jobs: python-version: 3.8 - name: Install lint tool run: | - python -m pip install --upgrade pip - pip install cmake-format pyyaml + PYTHON_CMD=python3 && $PYTHON_CMD -m pip --upgrade pip + PYTHON_CMD=python3 && $PYTHON_CMD -m pip install cmake-format pyyaml - name: Check cmakelist run: find . -name CMakeLists.txt |xargs cmake-format --check diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 4c3712e9dba6..bb776b79012e 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -94,7 +94,7 @@ jobs: unixodbc-dev - name: Install latest ninja - run: pip install ninja + run: PYTHON_CMD=python3 && $PYTHON_CMD -m pip install ninja - name: Configure if: matrix.language == 'c-cpp' diff --git a/.github/workflows/doc_checks.yml b/.github/workflows/doc_checks.yml index 545cb4eb5375..f5481f804c24 100644 --- a/.github/workflows/doc_checks.yml +++ b/.github/workflows/doc_checks.yml @@ -30,8 +30,8 @@ jobs: run: | apt update apt install -y libproj-dev swig - python3 -m pip install -r doc/requirements.txt - python3 -m pip install numpy setuptools + PYTHON_CMD=python3 && $PYTHON_CMD -m pip install -r doc/requirements.txt + PYTHON_CMD=python3 && $PYTHON_CMD -m pip install numpy setuptools pushd . mkdir build cd build @@ -52,21 +52,21 @@ jobs: cd swig/python python3 setup.py sdist cp dist/* /tmp/gdal.tar.gz - python3 -m pip install /tmp/gdal.tar.gz + PYTHON_CMD=python3 && $PYTHON_CMD -m pip install /tmp/gdal.tar.gz ldconfig popd - name: Update components shell: bash -l {0} run: | - pip install -U "sphinx-rtd-theme>=3.0.0" "sphinxcontrib-spelling>=8.0.0" + PYTHON_CMD=python3 && $PYTHON_CMD -m pip install -U "sphinx-rtd-theme>=3.0.0" "sphinxcontrib-spelling>=8.0.0" - name: Print versions shell: bash -l {0} run: | python3 --version sphinx-build --version - python3 -m pip list --not-required --format=columns + PYTHON_CMD=python3 && $PYTHON_CMD -m pip list --not-required --format=columns - name: Lint .rst files shell: bash -l {0} run: | diff --git a/.github/workflows/fedora_rawhide/Dockerfile.ci b/.github/workflows/fedora_rawhide/Dockerfile.ci index ca6bb876267b..1a768c94619b 100644 --- a/.github/workflows/fedora_rawhide/Dockerfile.ci +++ b/.github/workflows/fedora_rawhide/Dockerfile.ci @@ -26,5 +26,5 @@ RUN dnf install -y clang make diffutils ccache cmake \ COPY requirements.txt /tmp/ # One of the dependencies of jsonschema 4.18 is rpds_py which requires a Rust compiler -RUN python3 -m pip install "jsonschema<4.18" -RUN python3 -m pip install -r /tmp/requirements.txt +RUN PYTHON_CMD=python3 && $PYTHON_CMD -m pip install "jsonschema<4.18" +RUN PYTHON_CMD=python3 && $PYTHON_CMD -m pip install -r /tmp/requirements.txt diff --git a/.github/workflows/icc/Dockerfile.ci b/.github/workflows/icc/Dockerfile.ci index ece1570cfbad..09d3eb2756af 100644 --- a/.github/workflows/icc/Dockerfile.ci +++ b/.github/workflows/icc/Dockerfile.ci @@ -14,7 +14,7 @@ RUN wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/bb99984f-37 # cpython-38-x86_64-linux-gnu.so: undefined symbol: __intel_sse2_strcpy COPY requirements.txt /tmp/ -RUN python3 -m pip install -U -r /tmp/requirements.txt +RUN PYTHON_CMD=python3 && $PYTHON_CMD -m pip install -U -r /tmp/requirements.txt # Intel uses a script to set numerous environment variables. # Sourcing a script in the Dockerfile does not persist the variables diff --git a/.github/workflows/ubuntu_20.04/Dockerfile.ci b/.github/workflows/ubuntu_20.04/Dockerfile.ci index ebeab9487961..e72ea34c8741 100644 --- a/.github/workflows/ubuntu_20.04/Dockerfile.ci +++ b/.github/workflows/ubuntu_20.04/Dockerfile.ci @@ -200,7 +200,7 @@ RUN curl -v -j -k -s -L -H "Cookie: eula_3_2_agreed=tools.hana.ondemand.com/deve && rm hanaclient* ENV PATH=/usr/sap/hdbclient:$PATH -RUN python3 -m pip install hdbcli +RUN PYTHON_CMD=python3 && $PYTHON_CMD -m pip install hdbcli # Download and compile odbc-cpp-wrapper ENV ODBCCPP_VERSION=1.1 @@ -224,7 +224,7 @@ RUN wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | && apt-get install -y -V libarrow-dev libparquet-dev libarrow-dataset-dev \ && rm ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \ && rm -rf /var/lib/apt/lists/* \ - && python3 -m pip install pyarrow==11.0.0 + && PYTHON_CMD=python3 && $PYTHON_CMD -m pip install pyarrow==11.0.0 # Build libQB3 # Used by the MRF driver @@ -277,4 +277,4 @@ RUN if test "${OPENDRIVE_VERSION}" != ""; then ( \ RUN ldconfig COPY requirements.txt /tmp/ -RUN python3 -m pip install -U -r /tmp/requirements.txt +RUN PYTHON_CMD=python3 && $PYTHON_CMD -m pip install -U -r /tmp/requirements.txt diff --git a/.github/workflows/ubuntu_22.04/Dockerfile.ci b/.github/workflows/ubuntu_22.04/Dockerfile.ci index db60a6e92941..35ee72aaa549 100644 --- a/.github/workflows/ubuntu_22.04/Dockerfile.ci +++ b/.github/workflows/ubuntu_22.04/Dockerfile.ci @@ -137,5 +137,5 @@ RUN curl -L -O https://download.oracle.com/otn_software/linux/instantclient/1990 && apt-get install -y libaio1 COPY requirements.txt /tmp/ -RUN python3 -m pip install -U -r /tmp/requirements.txt +RUN PYTHON_CMD=python3 && $PYTHON_CMD -m pip install -U -r /tmp/requirements.txt diff --git a/.github/workflows/ubuntu_24.04/Dockerfile.ci b/.github/workflows/ubuntu_24.04/Dockerfile.ci index 30948d5e1ffe..b5e4bd88d3f7 100644 --- a/.github/workflows/ubuntu_24.04/Dockerfile.ci +++ b/.github/workflows/ubuntu_24.04/Dockerfile.ci @@ -162,13 +162,13 @@ RUN curl -L -O https://download.oracle.com/otn_software/linux/instantclient/1990 RUN ln -s /usr/lib/x86_64-linux-gnu/ogdi/4.1/libvrf.so /usr/lib/x86_64-linux-gnu COPY requirements.txt /tmp/ -RUN python3 -m pip install -U --break-system-packages -r /tmp/requirements.txt +RUN PYTHON_CMD=python3 && $PYTHON_CMD -m pip install -U --break-system-packages -r /tmp/requirements.txt # cfchecker requires udunits2 RUN apt-get install -y --allow-unauthenticated libudunits2-0 libudunits2-data -RUN python3 -m pip install --break-system-packages cfchecker +RUN PYTHON_CMD=python3 && $PYTHON_CMD -m pip install --break-system-packages cfchecker -RUN python3 -m pip install --break-system-packages fsspec +RUN PYTHON_CMD=python3 && $PYTHON_CMD -m pip install --break-system-packages fsspec # Manually install ADBC packages from Ubuntu 22.04 as there are no 24.04 packages at time of writing. RUN curl -LO -fsS https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-adbc/libadbc-driver-manager102_14-1_amd64.deb \ diff --git a/ci/travis/graviton2/script.sh b/ci/travis/graviton2/script.sh index c1d570a7f751..d0e9bdeaf546 100755 --- a/ci/travis/graviton2/script.sh +++ b/ci/travis/graviton2/script.sh @@ -9,7 +9,8 @@ export PYTEST="python3 -m pytest -vv -p no:sugar --color=no" (cd "$PWD/build" && make quicktest) # install test dependencies -sudo python3 -m pip install -U -r "$PWD/autotest/requirements.txt" +export PYTHON_CMD=python3 +sudo $PYTHON_CMD -m pip install -U -r "$PWD/autotest/requirements.txt" # Run all the Python autotests cd build diff --git a/ci/travis/s390x/script.sh b/ci/travis/s390x/script.sh index 6926178de487..8224d591ada6 100755 --- a/ci/travis/s390x/script.sh +++ b/ci/travis/s390x/script.sh @@ -9,7 +9,8 @@ export PYTEST="python3 -m pytest -vv -p no:sugar --color=no" (cd "$PWD/build" && make quicktest) # install test dependencies -sudo pip3 install -U -r "$PWD/autotest/requirements.txt" +export PIP_CMD=pip3 +sudo $PIP_CMD install -U -r "$PWD/autotest/requirements.txt" # Run all the Python autotests cd build diff --git a/scripts/vagrant/gdal.sh b/scripts/vagrant/gdal.sh index a52b662c4c01..5a5dc6bf02e2 100755 --- a/scripts/vagrant/gdal.sh +++ b/scripts/vagrant/gdal.sh @@ -31,4 +31,4 @@ cmake .. \ ninja -j6 sudo ninja install -python3 -m pip install -r ../autotest/requirements.txt +PYTHON_CMD=python3 && $PYTHON_CMD -m pip install -r ../autotest/requirements.txt diff --git a/swig/python/gdal-utils/test-ubuntu-install.sh.txt b/swig/python/gdal-utils/test-ubuntu-install.sh.txt index c70b619a17a3..3f626f9446e6 100644 --- a/swig/python/gdal-utils/test-ubuntu-install.sh.txt +++ b/swig/python/gdal-utils/test-ubuntu-install.sh.txt @@ -10,7 +10,7 @@ export CPLUS_INCLUDE_PATH=/usr/include/gdal export C_INCLUDE_PATH=/usr/include/gdal # Install python GDAL bindings -pip install gdal +PYTHON_CMD=python3 && $PYTHON_CMD -m pip install gdal # note: this could be `sudo apt-get install python3-gdal` instead but # pip method is preferred, better tested @@ -18,14 +18,14 @@ pip install gdal # (assumes git clone etc. are already done) cd swig/python/gdal-utils python setup.py bdist_wheel -pip install dist/gdal_utils-*.whl +PYTHON_CMD=python3 && $PYTHON_CMD -m pip install dist/gdal_utils-*.whl # ensure gdal source code tree installed gdal binaries are the same version git checkout v`gdal-config --version` # Test the scripts using autotest suite cd ../../../autotest/ -pip install -r requirements.txt +PYTHON_CMD=python3 && $PYTHON_CMD -m pip install -r requirements.txt pytest pyscripts/ # Todo: verify the console_scripts also work