From 697423762e503fbfef7db3a021ea41ab221d2ab1 Mon Sep 17 00:00:00 2001
From: Ken-ichi Ueda <kenichi.ueda@gmail.com>
Date: Wed, 11 Sep 2024 10:49:00 -0700
Subject: [PATCH] even more logging; disable non-contour jobs while debugging

---
 .github/workflows/build-pack.yml | 404 +++++++++++++++----------------
 elevation.py                     |  10 +-
 2 files changed, 210 insertions(+), 204 deletions(-)

diff --git a/.github/workflows/build-pack.yml b/.github/workflows/build-pack.yml
index 902aa59..ea23206 100644
--- a/.github/workflows/build-pack.yml
+++ b/.github/workflows/build-pack.yml
@@ -20,91 +20,91 @@ on:
         required: true
 
 jobs:
-  rocks:
-    name: Build rocks
-    runs-on: ubuntu-22.04
-    services:
-      postgres:
-        image: postgis/postgis:12-2.5
-        env:
-          POSTGRES_USER: underfoot
-          POSTGRES_PASSWORD: underfoot
-          POSTGRES_DB: underfoot
-        ports:
-          - 5432:5432
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install Ubuntu dependencies
-        run: ./.github/dependencies.sh
-
-      - name: Install Python dependencies
-        run: pip install -r requirements.txt && python setup.py
-
-      - id: cache-source-work-rocks
-        name: Cache source work for rocks
-        uses: actions/cache@v3
-        with:
-          path: sources/work-*
-          key: cache-source-work-rocks-2-${{ inputs.pack }}-${{ hashFiles(format('packs/{0}.json', inputs.pack)) }}
-
-      - name: Build rocks
-        run: |
-          python packs.py ${{ inputs.pack }} --only rocks ${{ inputs.use_pmtiles && '--use-pmtiles' }}
-        env:
-          PGHOST: 0.0.0.0
-          PGUSER: underfoot
-          PGPASSWORD: underfoot
-
-      - name: Upload rocks
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ inputs.pack }}-rocks
-          path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/"
-
-  water:
-    name: Build water
-    runs-on: ubuntu-22.04
-    services:
-      postgres:
-        image: postgis/postgis:12-2.5
-        env:
-          POSTGRES_USER: underfoot
-          POSTGRES_PASSWORD: underfoot
-          POSTGRES_DB: underfoot
-        ports:
-          - 5432:5432
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install Ubuntu dependencies
-        run: ./.github/dependencies.sh
-
-      - name: Install Python dependencies
-        run: pip install -r requirements.txt && python setup.py
-
-      - id: cache-source-work-water
-        name: Cache source work for water
-        uses: actions/cache@v3
-        with:
-          path: sources/work-*
-          key: cache-source-work-water-${{ inputs.pack }}-${{ hashFiles(format('packs/{0}.json', inputs.pack)) }}
-
-      - name: Build water
-        run: |
-          python packs.py ${{ inputs.pack }} --only water ${{ inputs.use_pmtiles && '--use-pmtiles' }}
-        env:
-          PGHOST: 0.0.0.0
-          PGUSER: underfoot
-          PGPASSWORD: underfoot
-
-      - name: Upload water
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ inputs.pack }}-water
-          path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/"
+  # rocks:
+  #   name: Build rocks
+  #   runs-on: ubuntu-22.04
+  #   services:
+  #     postgres:
+  #       image: postgis/postgis:12-2.5
+  #       env:
+  #         POSTGRES_USER: underfoot
+  #         POSTGRES_PASSWORD: underfoot
+  #         POSTGRES_DB: underfoot
+  #       ports:
+  #         - 5432:5432
+  #       options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+  #   steps:
+  #     - uses: actions/checkout@v4
+
+  #     - name: Install Ubuntu dependencies
+  #       run: ./.github/dependencies.sh
+
+  #     - name: Install Python dependencies
+  #       run: pip install -r requirements.txt && python setup.py
+
+  #     - id: cache-source-work-rocks
+  #       name: Cache source work for rocks
+  #       uses: actions/cache@v3
+  #       with:
+  #         path: sources/work-*
+  #         key: cache-source-work-rocks-2-${{ inputs.pack }}-${{ hashFiles(format('packs/{0}.json', inputs.pack)) }}
+
+  #     - name: Build rocks
+  #       run: |
+  #         python packs.py ${{ inputs.pack }} --only rocks ${{ inputs.use_pmtiles && '--use-pmtiles' }}
+  #       env:
+  #         PGHOST: 0.0.0.0
+  #         PGUSER: underfoot
+  #         PGPASSWORD: underfoot
+
+  #     - name: Upload rocks
+  #       uses: actions/upload-artifact@v4
+  #       with:
+  #         name: ${{ inputs.pack }}-rocks
+  #         path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/"
+
+  # water:
+  #   name: Build water
+  #   runs-on: ubuntu-22.04
+  #   services:
+  #     postgres:
+  #       image: postgis/postgis:12-2.5
+  #       env:
+  #         POSTGRES_USER: underfoot
+  #         POSTGRES_PASSWORD: underfoot
+  #         POSTGRES_DB: underfoot
+  #       ports:
+  #         - 5432:5432
+  #       options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+  #   steps:
+  #     - uses: actions/checkout@v4
+
+  #     - name: Install Ubuntu dependencies
+  #       run: ./.github/dependencies.sh
+
+  #     - name: Install Python dependencies
+  #       run: pip install -r requirements.txt && python setup.py
+
+  #     - id: cache-source-work-water
+  #       name: Cache source work for water
+  #       uses: actions/cache@v3
+  #       with:
+  #         path: sources/work-*
+  #         key: cache-source-work-water-${{ inputs.pack }}-${{ hashFiles(format('packs/{0}.json', inputs.pack)) }}
+
+  #     - name: Build water
+  #       run: |
+  #         python packs.py ${{ inputs.pack }} --only water ${{ inputs.use_pmtiles && '--use-pmtiles' }}
+  #       env:
+  #         PGHOST: 0.0.0.0
+  #         PGUSER: underfoot
+  #         PGPASSWORD: underfoot
+
+  #     - name: Upload water
+  #       uses: actions/upload-artifact@v4
+  #       with:
+  #         name: ${{ inputs.pack }}-water
+  #         path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/"
 
   contours:
     name: Build contours
@@ -149,120 +149,120 @@ jobs:
           name: ${{ inputs.pack }}-contours
           path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/"
 
-  osm:
-    name: Build ways & context
-    runs-on: ubuntu-22.04
-    services:
-      postgres:
-        image: postgis/postgis:12-2.5
-        env:
-          POSTGRES_USER: underfoot
-          POSTGRES_PASSWORD: underfoot
-          POSTGRES_DB: underfoot
-        ports:
-          - 5432:5432
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install Ubuntu dependencies
-        run: ./.github/dependencies.sh
-
-      - name: Install Python dependencies
-        run: pip install -r requirements.txt && python setup.py
-
-      - name: Get current year week
-        id: calc-year-week
-        run: echo "year-week=$(date +'%Y-%U')" >> $GITHUB_OUTPUT
-
-      - name: Get OSM URL
-        id: get-osm-url
-        run: UNDERFOOT_PACK=${{ inputs.pack }} && echo "osm-url=$(jq -c -r .osm packs/$UNDERFOOT_PACK.json)" >> $GITHUB_OUTPUT
-
-      - name: Cache OSM download
-        id: cache-osm
-        uses: actions/cache@v3
-        with:
-          path: "*.osm.pbf"
-          key: cache-osm-${{ steps.get-osm-url.outputs.osm-url }}-${{ steps.calc-year-week.outputs.year-week }}
-
-      - name: Build ways & context
-        run: |
-          python packs.py ${{ inputs.pack }} --only ways context ${{ inputs.use_pmtiles && '--use-pmtiles' }}
-        env:
-          PGHOST: 0.0.0.0
-          PGUSER: underfoot
-          PGPASSWORD: underfoot
-
-      - name: Upload ways
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ inputs.pack }}-ways
-          path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/"
-
-      - name: Upload context
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ inputs.pack }}-context
-          path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/"
-
-  upload-pack:
-    name: Archive and upload pack
-    runs-on: ubuntu-22.04
-    needs:
-      - rocks
-      - water
-      - contours
-      - osm
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install Ubuntu dependencies
-        run: ./.github/dependencies.sh
-
-      - name: Install Python dependencies
-        run: pip install -r requirements.txt && python setup.py
-
-      - name: Download all MBTiles artifacts
-        uses: actions/download-artifact@v4
-
-      - name: Zip up the pack
-        run: |
-          mkdir -p build/${{ inputs.pack }}
-          mv ${{ inputs.pack }}-rocks/* build/${{ inputs.pack }}/
-          mv ${{ inputs.pack }}-contours/* build/${{ inputs.pack }}/
-          mv ${{ inputs.pack }}-water/* build/${{ inputs.pack }}/
-          mv ${{ inputs.pack }}-ways/* build/${{ inputs.pack }}/
-          mv ${{ inputs.pack }}-context/* build/${{ inputs.pack }}/
-          cd build/
-          zip ${{ inputs.pack }}.${{ inputs.use_pmtiles && 'pmtiles.' || '' }}zip ${{ inputs.pack }}/*
-          cd ../
-
-      - name: Make the manifest
-        run: |
-          python packs.py manifest \
-            --s3-bucket-url https://static.underfoot.rocks \
-            --manifest-url https://static.underfoot.rocks/manifest.json
-
-      - name: Set up s3cmd for Digital Ocean
-        uses: s3-actions/s3cmd@v1.2.0
-        with:
-          provider: digitalocean
-          region: sfo2
-          access_key: ${{ secrets.DO_SPACES_KEY }}
-          secret_key: ${{ secrets.DO_SPACES_SECRET }}
-
-      - name: Upload pack to Digital Ocean
-        run: |
-          s3cmd put build/manifest.json s3://underfoot2/manifest.json
-          s3cmd setacl s3://underfoot2/manifest.json --acl-public
-          s3cmd put build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip s3://underfoot2/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip
-          s3cmd setacl s3://underfoot2/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip --acl-public
-
-      - name: Set up doctl
-        uses: digitalocean/action-doctl@v2
-        with:
-          token: ${{ secrets.DO_ACCESS_TOKEN }}
-
-      - name: Flush the CDN cache
-        run: doctl compute cdn flush ee6a9baf-c5bd-423d-bbcf-305e3a3f5e40
+  # osm:
+  #   name: Build ways & context
+  #   runs-on: ubuntu-22.04
+  #   services:
+  #     postgres:
+  #       image: postgis/postgis:12-2.5
+  #       env:
+  #         POSTGRES_USER: underfoot
+  #         POSTGRES_PASSWORD: underfoot
+  #         POSTGRES_DB: underfoot
+  #       ports:
+  #         - 5432:5432
+  #       options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+  #   steps:
+  #     - uses: actions/checkout@v4
+
+  #     - name: Install Ubuntu dependencies
+  #       run: ./.github/dependencies.sh
+
+  #     - name: Install Python dependencies
+  #       run: pip install -r requirements.txt && python setup.py
+
+  #     - name: Get current year week
+  #       id: calc-year-week
+  #       run: echo "year-week=$(date +'%Y-%U')" >> $GITHUB_OUTPUT
+
+  #     - name: Get OSM URL
+  #       id: get-osm-url
+  #       run: UNDERFOOT_PACK=${{ inputs.pack }} && echo "osm-url=$(jq -c -r .osm packs/$UNDERFOOT_PACK.json)" >> $GITHUB_OUTPUT
+
+  #     - name: Cache OSM download
+  #       id: cache-osm
+  #       uses: actions/cache@v3
+  #       with:
+  #         path: "*.osm.pbf"
+  #         key: cache-osm-${{ steps.get-osm-url.outputs.osm-url }}-${{ steps.calc-year-week.outputs.year-week }}
+
+  #     - name: Build ways & context
+  #       run: |
+  #         python packs.py ${{ inputs.pack }} --only ways context ${{ inputs.use_pmtiles && '--use-pmtiles' }}
+  #       env:
+  #         PGHOST: 0.0.0.0
+  #         PGUSER: underfoot
+  #         PGPASSWORD: underfoot
+
+  #     - name: Upload ways
+  #       uses: actions/upload-artifact@v4
+  #       with:
+  #         name: ${{ inputs.pack }}-ways
+  #         path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/"
+
+  #     - name: Upload context
+  #       uses: actions/upload-artifact@v4
+  #       with:
+  #         name: ${{ inputs.pack }}-context
+  #         path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/"
+
+  # upload-pack:
+  #   name: Archive and upload pack
+  #   runs-on: ubuntu-22.04
+  #   needs:
+  #     - rocks
+  #     - water
+  #     - contours
+  #     - osm
+  #   steps:
+  #     - uses: actions/checkout@v4
+
+  #     - name: Install Ubuntu dependencies
+  #       run: ./.github/dependencies.sh
+
+  #     - name: Install Python dependencies
+  #       run: pip install -r requirements.txt && python setup.py
+
+  #     - name: Download all MBTiles artifacts
+  #       uses: actions/download-artifact@v4
+
+  #     - name: Zip up the pack
+  #       run: |
+  #         mkdir -p build/${{ inputs.pack }}
+  #         mv ${{ inputs.pack }}-rocks/* build/${{ inputs.pack }}/
+  #         mv ${{ inputs.pack }}-contours/* build/${{ inputs.pack }}/
+  #         mv ${{ inputs.pack }}-water/* build/${{ inputs.pack }}/
+  #         mv ${{ inputs.pack }}-ways/* build/${{ inputs.pack }}/
+  #         mv ${{ inputs.pack }}-context/* build/${{ inputs.pack }}/
+  #         cd build/
+  #         zip ${{ inputs.pack }}.${{ inputs.use_pmtiles && 'pmtiles.' || '' }}zip ${{ inputs.pack }}/*
+  #         cd ../
+
+  #     - name: Make the manifest
+  #       run: |
+  #         python packs.py manifest \
+  #           --s3-bucket-url https://static.underfoot.rocks \
+  #           --manifest-url https://static.underfoot.rocks/manifest.json
+
+  #     - name: Set up s3cmd for Digital Ocean
+  #       uses: s3-actions/s3cmd@v1.2.0
+  #       with:
+  #         provider: digitalocean
+  #         region: sfo2
+  #         access_key: ${{ secrets.DO_SPACES_KEY }}
+  #         secret_key: ${{ secrets.DO_SPACES_SECRET }}
+
+  #     - name: Upload pack to Digital Ocean
+  #       run: |
+  #         s3cmd put build/manifest.json s3://underfoot2/manifest.json
+  #         s3cmd setacl s3://underfoot2/manifest.json --acl-public
+  #         s3cmd put build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip s3://underfoot2/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip
+  #         s3cmd setacl s3://underfoot2/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip --acl-public
+
+  #     - name: Set up doctl
+  #       uses: digitalocean/action-doctl@v2
+  #       with:
+  #         token: ${{ secrets.DO_ACCESS_TOKEN }}
+
+  #     - name: Flush the CDN cache
+  #       run: doctl compute cdn flush ee6a9baf-c5bd-423d-bbcf-305e3a3f5e40
diff --git a/elevation.py b/elevation.py
index 037fcf3..5429690 100644
--- a/elevation.py
+++ b/elevation.py
@@ -22,7 +22,8 @@
 
 
 TABLE_NAME = "contours"
-CACHE_DIR = tempfile.mkdtemp(suffix="-elevation-tiles")
+# CACHE_DIR = tempfile.mkdtemp(suffix="-elevation-tiles")
+CACHE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "elevation-tiles")
 
 
 # pylint: disable=invalid-name
@@ -72,8 +73,11 @@ async def cache_tile(tile, client, clean=False, max_retries=3, debug=False):
     """Caches a tile"""
     tile_path = f"{tile.z}/{tile.x}/{tile.y}.tif"
     url = f"https://s3.amazonaws.com/elevation-tiles-prod/geotiff/{tile_path}"
+    util.log(f"tile url: {url}")
     file_path = tile_file_path(tile.x, tile.y, tile.z)
+    util.log(f"file_path: {file_path}")
     dir_path = os.path.dirname(file_path)
+    util.log(f"dir_path: {dir_path}")
     os.makedirs(dir_path, exist_ok=True)
     if os.path.exists(file_path):
         util.log(f"cache_tile, path exists: {file_path}")
@@ -116,7 +120,7 @@ async def cache_tiles(tiles, clean=False):
     """Cache multiple tiles"""
     async with httpx.AsyncClient() as client:
         # using as_completed with tqdm (https://stackoverflow.com/a/37901797)
-        tasks = [cache_tile(tile, client, clean=clean) for tile in tiles]
+        tasks = [cache_tile(tile, client, clean=clean, debug=True) for tile in tiles]
         pbar = tqdm(
             asyncio.as_completed(tasks),
             total=len(tasks),
@@ -131,6 +135,8 @@ def make_contours_for_tile(tile, clean=False):
     """Make contours for a file"""
     # print("Making contours for {}".format(tile))
     tile_path = tile_file_path(tile.x, tile.y, tile.z)
+    dir_path = os.path.dirname(tile_path)
+    util.log(f"os.path.exists({dir_path}): {os.path.exists(dir_path)}")
     if not os.path.exists(tile_path):
         raise FileNotFoundError(f"Tile file does not exist at {tile_path}")
     merge_contours_path = tile_file_path(