From 91d9d64924bec0bfa262ff942eb68fa5c48ee698 Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Thu, 12 Sep 2024 14:42:22 -0700 Subject: [PATCH 01/14] Initial work --- .github/environment.yml | 28 ++++++++++++++ .../{dev_ci_cd.yml => dev_ci_cd_conda.yml} | 20 ++++++---- .github/workflows/dev_ci_cd_pip.yml | 37 +++++++++++++++++++ environment.yml | 4 +- setup.py | 3 +- 5 files changed, 81 insertions(+), 11 deletions(-) create mode 100644 .github/environment.yml rename .github/workflows/{dev_ci_cd.yml => dev_ci_cd_conda.yml} (61%) create mode 100644 .github/workflows/dev_ci_cd_pip.yml diff --git a/.github/environment.yml b/.github/environment.yml new file mode 100644 index 0000000..65bd6ab --- /dev/null +++ b/.github/environment.yml @@ -0,0 +1,28 @@ +name: cities-cif +channels: + - conda-forge +dependencies: + - python=3.10 + - earthengine-api=0.1.379 + - geocube=0.4.2 + - geopandas=0.14.4 + - rioxarray=0.15.0 + - odc-stac=0.3.8 + - pystac-client=0.7.5 + - pytest=7.4.3 + - xarray-spatial=0.3.7 + - xee=0.0.15 + - utm=0.7.0 + - osmnx=1.9.0 + - dask[complete]=2023.11.0 + - matplotlib=3.8.2 + - jupyterlab=4.0.10 + - s3fs=2024.5.0 + - geemap=0.32.0 + - pip=23.3.1 + - boto3=1.34.124 + - scikit-learn=1.5.1 + - scikit-image=0.24.0 + - exactextract=0.2.0 + - pip: + - overturemaps==0.6.0 \ No newline at end of file diff --git a/.github/workflows/dev_ci_cd.yml b/.github/workflows/dev_ci_cd_conda.yml similarity index 61% rename from .github/workflows/dev_ci_cd.yml rename to .github/workflows/dev_ci_cd_conda.yml index 6701372..31beaac 100644 --- a/.github/workflows/dev_ci_cd.yml +++ b/.github/workflows/dev_ci_cd_conda.yml @@ -1,4 +1,4 @@ -name: Dev CIF API CI/CD +name: Dev CIF API CI/CD Conda on: pull_request: @@ -13,25 +13,29 @@ jobs: max-parallel: 4 matrix: python-version: ["3.10"] - steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: + auto-update-conda: true python-version: ${{ matrix.python-version }} - name: Install Linux dependencies run: | sudo apt update sudo apt install -y gdal-bin libgdal-dev - - name: Install Packages + - name: Install Packages in environment.yml file + run: | + $CONDA/bin/conda env update --file=.github/environment.yml --name base + - name: Install other packages run: | - python -m pip install --upgrade pip - pip install -r .github/requirements.txt - pip install GDAL==`gdal-config --version` + $CONDA/bin/conda install gdal --yes + $CONDA/bin/conda upgrade numpy --yes + $CONDA/bin/conda install pytest --yes - name: Run Tests + shell: bash -l {0} env: GOOGLE_APPLICATION_USER: ${{ secrets.GOOGLE_APPLICATION_USER }} - GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} + GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} run: | - pytest tests + $CONDA/bin/pytest tests \ No newline at end of file diff --git a/.github/workflows/dev_ci_cd_pip.yml b/.github/workflows/dev_ci_cd_pip.yml new file mode 100644 index 0000000..add6674 --- /dev/null +++ b/.github/workflows/dev_ci_cd_pip.yml @@ -0,0 +1,37 @@ +#name: Dev CIF API CI/CD PIP +# +#on: +# pull_request: +# workflow_dispatch: +# +#permissions: +# contents: read +#jobs: +# build: +# runs-on: ubuntu-latest +# strategy: +# max-parallel: 4 +# matrix: +# python-version: ["3.10"] +# +# steps: +# - uses: actions/checkout@v4 +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v5 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Install Linux dependencies +# run: | +# sudo apt update +# sudo apt install -y gdal-bin libgdal-dev +# - name: Install Packages +# run: | +# python -m pip install --upgrade pip +# pip install -r .github/requirements.txt +# pip install GDAL==`gdal-config --version` +# - name: Run Tests +# env: +# GOOGLE_APPLICATION_USER: ${{ secrets.GOOGLE_APPLICATION_USER }} +# GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} +# run: | +# pytest tests diff --git a/environment.yml b/environment.yml index a064834..7d671ee 100644 --- a/environment.yml +++ b/environment.yml @@ -22,7 +22,7 @@ dependencies: - pip=23.3.1 - boto3=1.34.124 - scikit-learn=1.5.1 - - scikit-image==0.24.0 - - exactextract=0.2.0.dev252 + - scikit-image=0.24.0 + - exactextract=0.2.0 - pip: - overturemaps==0.6.0 diff --git a/setup.py b/setup.py index 7d9242c..74f0270 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ "exactextract<=0.2.0.dev252", "overturemaps", "scikit-learn>=1.5.1", - "scikit-image>=0.24.0" + "scikit-image>=0.24.0", + "exactextract>=0.2.0" ], ) From 646a6ff4b29a46b0966fb8e961f0dcdc6176c7ea Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Thu, 12 Sep 2024 15:46:54 -0700 Subject: [PATCH 02/14] changed to cities-cif env --- .github/workflows/dev_ci_cd_conda.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev_ci_cd_conda.yml b/.github/workflows/dev_ci_cd_conda.yml index 31beaac..2c015f9 100644 --- a/.github/workflows/dev_ci_cd_conda.yml +++ b/.github/workflows/dev_ci_cd_conda.yml @@ -26,12 +26,13 @@ jobs: sudo apt install -y gdal-bin libgdal-dev - name: Install Packages in environment.yml file run: | - $CONDA/bin/conda env update --file=.github/environment.yml --name base + $CONDA/bin/conda env update --file=.github/environment.yml - name: Install other packages run: | $CONDA/bin/conda install gdal --yes $CONDA/bin/conda upgrade numpy --yes $CONDA/bin/conda install pytest --yes + $CONDA/bin/conda activate cities-cif - name: Run Tests shell: bash -l {0} env: From 7130f272fe71644a46b03933e16a25209d914765 Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Thu, 12 Sep 2024 15:53:29 -0700 Subject: [PATCH 03/14] added conda init --- .github/workflows/dev_ci_cd_conda.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dev_ci_cd_conda.yml b/.github/workflows/dev_ci_cd_conda.yml index 2c015f9..3ad85eb 100644 --- a/.github/workflows/dev_ci_cd_conda.yml +++ b/.github/workflows/dev_ci_cd_conda.yml @@ -32,6 +32,7 @@ jobs: $CONDA/bin/conda install gdal --yes $CONDA/bin/conda upgrade numpy --yes $CONDA/bin/conda install pytest --yes + $CONDA/bin/conda init $CONDA/bin/conda activate cities-cif - name: Run Tests shell: bash -l {0} From 5fbc347ee1a330e5f7cad4ce5637754906feb336 Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Thu, 12 Sep 2024 16:26:53 -0700 Subject: [PATCH 04/14] created activate environment section --- .github/workflows/dev_ci_cd_conda.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/dev_ci_cd_conda.yml b/.github/workflows/dev_ci_cd_conda.yml index 3ad85eb..910dfe0 100644 --- a/.github/workflows/dev_ci_cd_conda.yml +++ b/.github/workflows/dev_ci_cd_conda.yml @@ -32,6 +32,9 @@ jobs: $CONDA/bin/conda install gdal --yes $CONDA/bin/conda upgrade numpy --yes $CONDA/bin/conda install pytest --yes + - name: Activate environment + run: | + source ~/.bashrc $CONDA/bin/conda init $CONDA/bin/conda activate cities-cif - name: Run Tests From 362b96dbfe3f205fe57ea30a7807c0d23bcf6094 Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Thu, 12 Sep 2024 16:34:40 -0700 Subject: [PATCH 05/14] trying other was to restart shell --- .github/workflows/dev_ci_cd_conda.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev_ci_cd_conda.yml b/.github/workflows/dev_ci_cd_conda.yml index 910dfe0..b4fff9c 100644 --- a/.github/workflows/dev_ci_cd_conda.yml +++ b/.github/workflows/dev_ci_cd_conda.yml @@ -32,9 +32,10 @@ jobs: $CONDA/bin/conda install gdal --yes $CONDA/bin/conda upgrade numpy --yes $CONDA/bin/conda install pytest --yes + source /home/runner/.bashrc - name: Activate environment + shell: bash -l {0} run: | - source ~/.bashrc $CONDA/bin/conda init $CONDA/bin/conda activate cities-cif - name: Run Tests From e86f01a15c11a43b45634085f47167a0af849a34 Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Thu, 12 Sep 2024 16:43:24 -0700 Subject: [PATCH 06/14] more attempts to activate --- .github/workflows/dev_ci_cd_conda.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dev_ci_cd_conda.yml b/.github/workflows/dev_ci_cd_conda.yml index b4fff9c..f38f8f4 100644 --- a/.github/workflows/dev_ci_cd_conda.yml +++ b/.github/workflows/dev_ci_cd_conda.yml @@ -32,12 +32,13 @@ jobs: $CONDA/bin/conda install gdal --yes $CONDA/bin/conda upgrade numpy --yes $CONDA/bin/conda install pytest --yes - source /home/runner/.bashrc - name: Activate environment shell: bash -l {0} run: | - $CONDA/bin/conda init - $CONDA/bin/conda activate cities-cif + source ~/.bashrc + conda init + conda activate base + conda activate cities-cif - name: Run Tests shell: bash -l {0} env: From c9eea3afdbbf4eeee7e0f590ab9b52010e9fe875 Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Thu, 12 Sep 2024 16:55:54 -0700 Subject: [PATCH 07/14] more attempts to activate --- .github/workflows/dev_ci_cd_conda.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dev_ci_cd_conda.yml b/.github/workflows/dev_ci_cd_conda.yml index f38f8f4..28940ca 100644 --- a/.github/workflows/dev_ci_cd_conda.yml +++ b/.github/workflows/dev_ci_cd_conda.yml @@ -32,17 +32,17 @@ jobs: $CONDA/bin/conda install gdal --yes $CONDA/bin/conda upgrade numpy --yes $CONDA/bin/conda install pytest --yes + source ~/.bashrc - name: Activate environment - shell: bash -l {0} run: | - source ~/.bashrc + shell: bash -l {0} conda init conda activate base conda activate cities-cif - name: Run Tests - shell: bash -l {0} env: GOOGLE_APPLICATION_USER: ${{ secrets.GOOGLE_APPLICATION_USER }} GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} run: | + shell: bash -l {0} $CONDA/bin/pytest tests \ No newline at end of file From 59cd65cb0b0edc8804c61e746c44cde978846c44 Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Thu, 12 Sep 2024 17:17:16 -0700 Subject: [PATCH 08/14] more attempts to activate --- .github/workflows/dev_ci_cd_conda.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev_ci_cd_conda.yml b/.github/workflows/dev_ci_cd_conda.yml index 28940ca..6184790 100644 --- a/.github/workflows/dev_ci_cd_conda.yml +++ b/.github/workflows/dev_ci_cd_conda.yml @@ -34,15 +34,15 @@ jobs: $CONDA/bin/conda install pytest --yes source ~/.bashrc - name: Activate environment + shell: bash -l {0} run: | - shell: bash -l {0} conda init conda activate base conda activate cities-cif - name: Run Tests + shell: bash -l {0} env: GOOGLE_APPLICATION_USER: ${{ secrets.GOOGLE_APPLICATION_USER }} GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} run: | - shell: bash -l {0} $CONDA/bin/pytest tests \ No newline at end of file From e0ac5ac02e3a315adf16f6a75167a52df578d2d4 Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Thu, 12 Sep 2024 17:33:40 -0700 Subject: [PATCH 09/14] large rewrite --- .github/workflows/dev_ci_cd_conda.yml | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/workflows/dev_ci_cd_conda.yml b/.github/workflows/dev_ci_cd_conda.yml index 6184790..a9e8008 100644 --- a/.github/workflows/dev_ci_cd_conda.yml +++ b/.github/workflows/dev_ci_cd_conda.yml @@ -15,6 +15,12 @@ jobs: python-version: ["3.10"] steps: - uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v2 + with: + activate-environment: cities-cif + environment-file: .github/environment.yml + python-version: ${{ matrix.python-version }} + auto-activate-base: true - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -24,25 +30,15 @@ jobs: run: | sudo apt update sudo apt install -y gdal-bin libgdal-dev - - name: Install Packages in environment.yml file - run: | - $CONDA/bin/conda env update --file=.github/environment.yml - name: Install other packages run: | - $CONDA/bin/conda install gdal --yes - $CONDA/bin/conda upgrade numpy --yes - $CONDA/bin/conda install pytest --yes - source ~/.bashrc - - name: Activate environment - shell: bash -l {0} - run: | - conda init - conda activate base - conda activate cities-cif + conda install gdal --yes + conda upgrade numpy --yes + conda install pytest --yes - name: Run Tests - shell: bash -l {0} + shell: bash -l env: GOOGLE_APPLICATION_USER: ${{ secrets.GOOGLE_APPLICATION_USER }} GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} run: | - $CONDA/bin/pytest tests \ No newline at end of file + pytest tests \ No newline at end of file From 0eb29ccfa18444ce095b5250e9c100804bd0daf5 Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Thu, 12 Sep 2024 17:47:41 -0700 Subject: [PATCH 10/14] more attempts --- .github/workflows/dev_ci_cd_conda.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev_ci_cd_conda.yml b/.github/workflows/dev_ci_cd_conda.yml index a9e8008..46ad75f 100644 --- a/.github/workflows/dev_ci_cd_conda.yml +++ b/.github/workflows/dev_ci_cd_conda.yml @@ -9,6 +9,9 @@ permissions: jobs: build: runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} strategy: max-parallel: 4 matrix: @@ -21,10 +24,10 @@ jobs: environment-file: .github/environment.yml python-version: ${{ matrix.python-version }} auto-activate-base: true + auto-update-conda: true - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: - auto-update-conda: true python-version: ${{ matrix.python-version }} - name: Install Linux dependencies run: | @@ -32,11 +35,8 @@ jobs: sudo apt install -y gdal-bin libgdal-dev - name: Install other packages run: | - conda install gdal --yes - conda upgrade numpy --yes conda install pytest --yes - name: Run Tests - shell: bash -l env: GOOGLE_APPLICATION_USER: ${{ secrets.GOOGLE_APPLICATION_USER }} GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} From f55012eca0140fbc13ec11570a3d2f7052ac37c8 Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Thu, 12 Sep 2024 18:49:20 -0700 Subject: [PATCH 11/14] upgrading earthengine api --- .github/environment.yml | 2 +- environment.yml | 2 +- setup.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/environment.yml b/.github/environment.yml index 65bd6ab..3dbb817 100644 --- a/.github/environment.yml +++ b/.github/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge dependencies: - python=3.10 - - earthengine-api=0.1.379 + - earthengine-api=0.1.411 - geocube=0.4.2 - geopandas=0.14.4 - rioxarray=0.15.0 diff --git a/environment.yml b/environment.yml index 7d671ee..0d00195 100644 --- a/environment.yml +++ b/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge dependencies: - python=3.10 - - earthengine-api=0.1.379 + - earthengine-api=0.1.411 - geocube=0.4.2 - geopandas=0.14.4 - rioxarray=0.15.0 diff --git a/setup.py b/setup.py index 74f0270..2c6cceb 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,6 @@ "s3fs", "dask>=2023.11.0", "boto3", - "exactextract<=0.2.0.dev252", "overturemaps", "scikit-learn>=1.5.1", "scikit-image>=0.24.0", From 112f5490661cedb516dfde587d9a840b2d18b6bc Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Thu, 12 Sep 2024 21:23:26 -0700 Subject: [PATCH 12/14] refine yml --- .github/requirements.txt | 4 ++-- .github/workflows/dev_ci_cd_conda.yml | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/requirements.txt b/.github/requirements.txt index f5069a9..ba2c940 100644 --- a/.github/requirements.txt +++ b/.github/requirements.txt @@ -1,4 +1,4 @@ -earthengine-api==0.1.408 +earthengine-api==0.1.411 geocube==0.4.2 geopandas==0.14.1 rioxarray==0.15.0 @@ -18,4 +18,4 @@ boto3==1.34.124 scikit-learn==1.5.1 scikit-image==0.24.0 overturemaps==0.6.0 -exactextract==0.2.0.dev252 +exactextract==0.2.0 diff --git a/.github/workflows/dev_ci_cd_conda.yml b/.github/workflows/dev_ci_cd_conda.yml index 46ad75f..ecd2ebc 100644 --- a/.github/workflows/dev_ci_cd_conda.yml +++ b/.github/workflows/dev_ci_cd_conda.yml @@ -23,7 +23,6 @@ jobs: activate-environment: cities-cif environment-file: .github/environment.yml python-version: ${{ matrix.python-version }} - auto-activate-base: true auto-update-conda: true - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 From df14ab6441c158c76c791a8bf4e62d78cc68d9c9 Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Thu, 12 Sep 2024 21:31:22 -0700 Subject: [PATCH 13/14] thinning yml --- .github/workflows/dev_ci_cd_conda.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/dev_ci_cd_conda.yml b/.github/workflows/dev_ci_cd_conda.yml index ecd2ebc..45cb64d 100644 --- a/.github/workflows/dev_ci_cd_conda.yml +++ b/.github/workflows/dev_ci_cd_conda.yml @@ -24,10 +24,6 @@ jobs: environment-file: .github/environment.yml python-version: ${{ matrix.python-version }} auto-update-conda: true - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - name: Install Linux dependencies run: | sudo apt update From 5ea89fdd9404463e033b180f12f6de62a7faedbf Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Thu, 12 Sep 2024 22:40:33 -0700 Subject: [PATCH 14/14] removed pip files --- .github/requirements.txt | 21 ---------------- .github/workflows/dev_ci_cd_pip.yml | 37 ----------------------------- 2 files changed, 58 deletions(-) delete mode 100644 .github/requirements.txt delete mode 100644 .github/workflows/dev_ci_cd_pip.yml diff --git a/.github/requirements.txt b/.github/requirements.txt deleted file mode 100644 index ba2c940..0000000 --- a/.github/requirements.txt +++ /dev/null @@ -1,21 +0,0 @@ -earthengine-api==0.1.411 -geocube==0.4.2 -geopandas==0.14.1 -rioxarray==0.15.0 -odc-stac==0.3.8 -pystac-client==0.7.5 -pytest==7.4.3 -xarray-spatial==0.3.7 -xee==0.0.15 -utm==0.7.0 -osmnx==1.9.3 -dask[complete]==2023.11.0 -matplotlib==3.8.2 -s3fs==2024.5.0 -geemap==0.32.0 -pip==23.3.1 -boto3==1.34.124 -scikit-learn==1.5.1 -scikit-image==0.24.0 -overturemaps==0.6.0 -exactextract==0.2.0 diff --git a/.github/workflows/dev_ci_cd_pip.yml b/.github/workflows/dev_ci_cd_pip.yml deleted file mode 100644 index add6674..0000000 --- a/.github/workflows/dev_ci_cd_pip.yml +++ /dev/null @@ -1,37 +0,0 @@ -#name: Dev CIF API CI/CD PIP -# -#on: -# pull_request: -# workflow_dispatch: -# -#permissions: -# contents: read -#jobs: -# build: -# runs-on: ubuntu-latest -# strategy: -# max-parallel: 4 -# matrix: -# python-version: ["3.10"] -# -# steps: -# - uses: actions/checkout@v4 -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v5 -# with: -# python-version: ${{ matrix.python-version }} -# - name: Install Linux dependencies -# run: | -# sudo apt update -# sudo apt install -y gdal-bin libgdal-dev -# - name: Install Packages -# run: | -# python -m pip install --upgrade pip -# pip install -r .github/requirements.txt -# pip install GDAL==`gdal-config --version` -# - name: Run Tests -# env: -# GOOGLE_APPLICATION_USER: ${{ secrets.GOOGLE_APPLICATION_USER }} -# GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} -# run: | -# pytest tests