feat: soil id algorithm dependency (with Debian 12) (#1290)

* fix: add soil_id dependency * chore: update requirements.txt * fix: basic soil id test * build: add download-soil-data task to Makefile * docs: add soil id data to readme * fix: rework Dockerfile for Debian 12
techmatters · May 14, 2024 · 7ec6996 · 7ec6996
1 parent 1b2f94d
commit 7ec6996
Show file tree

Hide file tree

Showing 11 changed files with 400 additions and 28 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -23,8 +23,7 @@ jobs:
           python-version: "3.12.3"
 
       - name: Install Python dependencies for CI
-        run: |
-          pip install -r requirements-dev.txt
+        run: make install-dev
 
       - name: Build base Docker image
         run: |
@@ -33,8 +32,7 @@ jobs:
           make build DC_ENV=ci
 
       - name: Run lint checks
-        run: |
-          make lint
+        run: make lint
 
   build-and-test:
     runs-on: ubuntu-latest
@@ -50,6 +48,24 @@ jobs:
           echo "SENTRY_DSN=${{ secrets.SENTRY_DSN }}" >> .env
           make build DC_ENV=ci
 
-      - name: Run tests using built Docker image
+      - name: Restore data cache
+        uses: actions/cache@v4
+        with:
+          path: Data/
+          key: ${{ runner.os }}-data-${{ hashFiles('Data/*') }}
+          restore-keys: ${{ runner.os }}-data-
+
+      - name: Setup Python
+        if: ${{ hashFiles('Data/*') == '' }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12.3"
+
+      - name: Install Python dependencies and download data
+        if: ${{ hashFiles('Data/*') == '' }}
         run: |
-          make test-ci DC_ENV=ci
+          make install-dev
+          make download-soil-data
+
+      - name: Run tests using built Docker image
+        run: make test-ci DC_ENV=ci
diff --git a/Dockerfile b/Dockerfile
@@ -1,11 +1,17 @@
-FROM python:3.12.3-slim-bullseye
+FROM python:3.12.3-slim-bookworm
 
 RUN adduser --disabled-password terraso
 
 ENV PATH /home/terraso/.local/bin:$PATH
 # see https://github.com/aws/aws-cli/tags for list of versions
 ENV AWS_CLI_VERSION 2.8.12
 
+# Add testing sources and pin the GDAL packages to testing
+# Allows us to get 3.8.x versions of GDAL
+RUN sed 's/bookworm/testing/g' /etc/apt/sources.list.d/debian.sources >  /etc/apt/sources.list.d/testing.sources
+
+RUN echo 'Package: libgdal-dev gdal-bin\nPin: release a=testing\nPin-Priority: 900' > /etc/apt/preferences.d/gdal-testing
+
 RUN apt-get update && \
     apt-get install -q -y --no-install-recommends \
                      build-essential libpq-dev dnsutils libmagic-dev mailcap \

diff --git a/Makefile b/Makefile
@@ -112,6 +112,18 @@ test-ci: clean
 bash:
 	$(DC_RUN_CMD) bash
 
+# Donwload Munsell CSV, SHX, SHP, SBX, SBN, PRJ, DBF
+download-soil-data:
+	mkdir -p Data
+	cd Data; \
+	gdown 1tN23iVe6X1fcomcfveVp4w3Pwd0HJuTe; \
+	gdown 1WUa9e3vTWPi6G8h4OI3CBUZP5y7tf1Li; \
+	gdown 1l9MxC0xENGmI_NmGlBY74EtlD6SZid_a; \
+	gdown 1asGnnqe0zI2v8xuOszlsNmZkOSl7cJ2n; \
+	gdown 185Qjb9pJJn4AzOissiTz283tINrDqgI0; \
+	gdown 1P3xl1YRlfcMjfO_4PM39tkrrlL3hoLzv; \
+	gdown 1K0GkqxhZiVUND6yfFmaI7tYanLktekyp \
+
 ${VIRTUAL_ENV}/scripts/black:
 	pip install black
 

diff --git a/README.md b/README.md
@@ -65,14 +65,20 @@ need to attach to the application running container.
 
 So, assuming that the application is running with `make run`:
 
+List the running containers
+
 ```sh
-# List the running containers
 $ docker ps
-# Get the id of the web container before next step
+```
+
+Get the id of the web container before next step
+
+```sh
 $ docker attach <web-container-id>
-# This will give you access to the web running container
 ```
 
+This will give you access to the web running container
+
 Make the application request call that will pass on breakpoint, like
 calling an API or clicking in some button. As soon as the process get to
 the break point, the attached shell should open the Python debugger. To
@@ -92,6 +98,12 @@ Import landscape boundaries geodata:
 $ python terraso_backend/manage.py load_landscapes_geojson --airtable_api_key xxxxx
 ```
 
+Download Soil ID data:
+
+```sh
+$ make download-soil-data
+```
+
 ## Reset the database
 
 You can reset the database back to its default state:

diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml
@@ -5,7 +5,10 @@ services:
     extends:
       file: docker-compose.base.yml
       service: web
-
+    volumes:
+      - type: bind
+        source: /home/runner/work/terraso-backend/terraso-backend/Data
+        target: /app/Data
     depends_on:
       db:
         condition: service_healthy

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -12,6 +12,10 @@ asttokens==2.4.1 \
     --hash=sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24 \
     --hash=sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0
     # via stack-data
+beautifulsoup4==4.12.3 \
+    --hash=sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051 \
+    --hash=sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed
+    # via gdown
 black==24.4.2 \
     --hash=sha256:257d724c2c9b1660f353b36c802ccece186a30accc7742c176d29c146df6e474 \
     --hash=sha256:37aae07b029fa0174d39daf02748b379399b909652a806e5708199bd93899da1 \
@@ -36,9 +40,9 @@ black==24.4.2 \
     --hash=sha256:eaea3008c281f1038edb473c1aa8ed8143a5535ff18f978a318f10302b254063 \
     --hash=sha256:ef703f83fc32e131e9bcc0a5094cfe85599e7109f896fe8bc96cc402f3eb4b6e
     # via -r requirements/dev.in
-boto3==1.34.96 \
-    --hash=sha256:42ea7d46688e7cb27259780b9da2cddcfaf2763ff5d327f4d54eac12edba8e72 \
-    --hash=sha256:fe3d039631074a96374a354764641b6623036b6ea15381b8a04ac0a193b8c1e0
+boto3==1.34.103 \
+    --hash=sha256:58d097241f3895c4a4c80c9e606689c6e06d77f55f9f53a4cc02dee7e03938b9 \
+    --hash=sha256:59b6499f1bb423dd99de6566a20d0a7cf1a5476824be3a792290fd86600e8365
     # via moto
 botocore==1.34.103 \
     --hash=sha256:0330d139f18f78d38127e65361859e24ebd6a8bcba184f903c01bb999a3fa431 \
@@ -204,7 +208,7 @@ click==8.1.7 \
     --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \
     --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de
     # via black
-coverage[toml]==7.5.1 \
+coverage==7.5.1 \
     --hash=sha256:0646599e9b139988b63704d704af8e8df7fa4cbc4a1f33df69d97f36cb0a38de \
     --hash=sha256:0cdcbc320b14c3e5877ee79e649677cb7d89ef588852e9583e6b24c2e5072661 \
     --hash=sha256:0d0a0f5e06881ecedfe6f3dd2f56dcb057b6dbeb3327fd32d4b12854df36bf26 \
@@ -306,6 +310,10 @@ faker==12.0.1 \
     --hash=sha256:1dc2811f20e163892fefe7006f2ce00778f8099a40aee265bfa60a13400de63d \
     --hash=sha256:aa7103805ae793277abbb85da9f6f05e76a1a295a9384a8e17c2fba2b3a690cb
     # via mixer
+filelock==3.14.0 \
+    --hash=sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f \
+    --hash=sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a
+    # via gdown
 flake8==7.0.0 \
     --hash=sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132 \
     --hash=sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3
@@ -314,6 +322,10 @@ freezegun==1.5.1 \
     --hash=sha256:b29dedfcda6d5e8e083ce71b2b542753ad48cfec44037b3fc79702e2980a89e9 \
     --hash=sha256:bf111d7138a8abe55ab48a71755673dbaa4ab87f4cff5634a4442dfec34c15f1
     # via -r requirements/dev.in
+gdown==5.2.0 \
+    --hash=sha256:2145165062d85520a3cd98b356c9ed522c5e7984d408535409fd46f94defc787 \
+    --hash=sha256:33083832d82b1101bdd0e9df3edd0fbc0e1c5f14c9d8c38d2a35bf1683b526d6
+    # via -r requirements/dev.in
 h11==0.14.0 \
     --hash=sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d \
     --hash=sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761
@@ -509,6 +521,11 @@ pygments==2.18.0 \
     --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \
     --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a
     # via ipython
+pysocks==1.7.1 \
+    --hash=sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299 \
+    --hash=sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5 \
+    --hash=sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0
+    # via requests
 pytest==8.2.0 \
     --hash=sha256:1733f0620f6cda4095bbf0d9ff8022486e91892245bb9e7d5542c018f612f233 \
     --hash=sha256:d507d4482197eac0ba2bae2e9babf0672eb333017bcedaa5fb1a3d42c1174b3f
@@ -589,6 +606,7 @@ requests==2.31.0 \
     --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \
     --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1
     # via
+    #   gdown
     #   moto
     #   responses
 responses==0.25.0 \
@@ -615,10 +633,18 @@ sniffio==1.3.1 \
     # via
     #   anyio
     #   httpx
+soupsieve==2.5 \
+    --hash=sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690 \
+    --hash=sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7
+    # via beautifulsoup4
 stack-data==0.6.3 \
     --hash=sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9 \
     --hash=sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695
     # via ipython
+tqdm==4.66.4 \
+    --hash=sha256:b75ca56b413b030bc3f00af51fd2c1a1a5eac6a0c1cca83cbb37a5c52abce644 \
+    --hash=sha256:e4d936c9de8727928f3be6079590e97d9abfe8d39a590be678eb5919ffc186bb
+    # via gdown
 traitlets==5.14.3 \
     --hash=sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7 \
     --hash=sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f