From 697423762e503fbfef7db3a021ea41ab221d2ab1 Mon Sep 17 00:00:00 2001 From: Ken-ichi Ueda <kenichi.ueda@gmail.com> Date: Wed, 11 Sep 2024 10:49:00 -0700 Subject: [PATCH] even more logging; disable non-contour jobs while debugging --- .github/workflows/build-pack.yml | 404 +++++++++++++++---------------- elevation.py | 10 +- 2 files changed, 210 insertions(+), 204 deletions(-) diff --git a/.github/workflows/build-pack.yml b/.github/workflows/build-pack.yml index 902aa59..ea23206 100644 --- a/.github/workflows/build-pack.yml +++ b/.github/workflows/build-pack.yml @@ -20,91 +20,91 @@ on: required: true jobs: - rocks: - name: Build rocks - runs-on: ubuntu-22.04 - services: - postgres: - image: postgis/postgis:12-2.5 - env: - POSTGRES_USER: underfoot - POSTGRES_PASSWORD: underfoot - POSTGRES_DB: underfoot - ports: - - 5432:5432 - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - steps: - - uses: actions/checkout@v4 - - - name: Install Ubuntu dependencies - run: ./.github/dependencies.sh - - - name: Install Python dependencies - run: pip install -r requirements.txt && python setup.py - - - id: cache-source-work-rocks - name: Cache source work for rocks - uses: actions/cache@v3 - with: - path: sources/work-* - key: cache-source-work-rocks-2-${{ inputs.pack }}-${{ hashFiles(format('packs/{0}.json', inputs.pack)) }} - - - name: Build rocks - run: | - python packs.py ${{ inputs.pack }} --only rocks ${{ inputs.use_pmtiles && '--use-pmtiles' }} - env: - PGHOST: 0.0.0.0 - PGUSER: underfoot - PGPASSWORD: underfoot - - - name: Upload rocks - uses: actions/upload-artifact@v4 - with: - name: ${{ inputs.pack }}-rocks - path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" - - water: - name: Build water - runs-on: ubuntu-22.04 - services: - postgres: - image: postgis/postgis:12-2.5 - env: - POSTGRES_USER: underfoot - POSTGRES_PASSWORD: underfoot - POSTGRES_DB: underfoot - ports: - - 5432:5432 - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - steps: - - uses: actions/checkout@v4 - - - name: Install Ubuntu dependencies - run: ./.github/dependencies.sh - - - name: Install Python dependencies - run: pip install -r requirements.txt && python setup.py - - - id: cache-source-work-water - name: Cache source work for water - uses: actions/cache@v3 - with: - path: sources/work-* - key: cache-source-work-water-${{ inputs.pack }}-${{ hashFiles(format('packs/{0}.json', inputs.pack)) }} - - - name: Build water - run: | - python packs.py ${{ inputs.pack }} --only water ${{ inputs.use_pmtiles && '--use-pmtiles' }} - env: - PGHOST: 0.0.0.0 - PGUSER: underfoot - PGPASSWORD: underfoot - - - name: Upload water - uses: actions/upload-artifact@v4 - with: - name: ${{ inputs.pack }}-water - path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" + # rocks: + # name: Build rocks + # runs-on: ubuntu-22.04 + # services: + # postgres: + # image: postgis/postgis:12-2.5 + # env: + # POSTGRES_USER: underfoot + # POSTGRES_PASSWORD: underfoot + # POSTGRES_DB: underfoot + # ports: + # - 5432:5432 + # options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + # steps: + # - uses: actions/checkout@v4 + + # - name: Install Ubuntu dependencies + # run: ./.github/dependencies.sh + + # - name: Install Python dependencies + # run: pip install -r requirements.txt && python setup.py + + # - id: cache-source-work-rocks + # name: Cache source work for rocks + # uses: actions/cache@v3 + # with: + # path: sources/work-* + # key: cache-source-work-rocks-2-${{ inputs.pack }}-${{ hashFiles(format('packs/{0}.json', inputs.pack)) }} + + # - name: Build rocks + # run: | + # python packs.py ${{ inputs.pack }} --only rocks ${{ inputs.use_pmtiles && '--use-pmtiles' }} + # env: + # PGHOST: 0.0.0.0 + # PGUSER: underfoot + # PGPASSWORD: underfoot + + # - name: Upload rocks + # uses: actions/upload-artifact@v4 + # with: + # name: ${{ inputs.pack }}-rocks + # path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" + + # water: + # name: Build water + # runs-on: ubuntu-22.04 + # services: + # postgres: + # image: postgis/postgis:12-2.5 + # env: + # POSTGRES_USER: underfoot + # POSTGRES_PASSWORD: underfoot + # POSTGRES_DB: underfoot + # ports: + # - 5432:5432 + # options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + # steps: + # - uses: actions/checkout@v4 + + # - name: Install Ubuntu dependencies + # run: ./.github/dependencies.sh + + # - name: Install Python dependencies + # run: pip install -r requirements.txt && python setup.py + + # - id: cache-source-work-water + # name: Cache source work for water + # uses: actions/cache@v3 + # with: + # path: sources/work-* + # key: cache-source-work-water-${{ inputs.pack }}-${{ hashFiles(format('packs/{0}.json', inputs.pack)) }} + + # - name: Build water + # run: | + # python packs.py ${{ inputs.pack }} --only water ${{ inputs.use_pmtiles && '--use-pmtiles' }} + # env: + # PGHOST: 0.0.0.0 + # PGUSER: underfoot + # PGPASSWORD: underfoot + + # - name: Upload water + # uses: actions/upload-artifact@v4 + # with: + # name: ${{ inputs.pack }}-water + # path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" contours: name: Build contours @@ -149,120 +149,120 @@ jobs: name: ${{ inputs.pack }}-contours path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" - osm: - name: Build ways & context - runs-on: ubuntu-22.04 - services: - postgres: - image: postgis/postgis:12-2.5 - env: - POSTGRES_USER: underfoot - POSTGRES_PASSWORD: underfoot - POSTGRES_DB: underfoot - ports: - - 5432:5432 - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - steps: - - uses: actions/checkout@v4 - - - name: Install Ubuntu dependencies - run: ./.github/dependencies.sh - - - name: Install Python dependencies - run: pip install -r requirements.txt && python setup.py - - - name: Get current year week - id: calc-year-week - run: echo "year-week=$(date +'%Y-%U')" >> $GITHUB_OUTPUT - - - name: Get OSM URL - id: get-osm-url - run: UNDERFOOT_PACK=${{ inputs.pack }} && echo "osm-url=$(jq -c -r .osm packs/$UNDERFOOT_PACK.json)" >> $GITHUB_OUTPUT - - - name: Cache OSM download - id: cache-osm - uses: actions/cache@v3 - with: - path: "*.osm.pbf" - key: cache-osm-${{ steps.get-osm-url.outputs.osm-url }}-${{ steps.calc-year-week.outputs.year-week }} - - - name: Build ways & context - run: | - python packs.py ${{ inputs.pack }} --only ways context ${{ inputs.use_pmtiles && '--use-pmtiles' }} - env: - PGHOST: 0.0.0.0 - PGUSER: underfoot - PGPASSWORD: underfoot - - - name: Upload ways - uses: actions/upload-artifact@v4 - with: - name: ${{ inputs.pack }}-ways - path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" - - - name: Upload context - uses: actions/upload-artifact@v4 - with: - name: ${{ inputs.pack }}-context - path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" - - upload-pack: - name: Archive and upload pack - runs-on: ubuntu-22.04 - needs: - - rocks - - water - - contours - - osm - steps: - - uses: actions/checkout@v4 - - - name: Install Ubuntu dependencies - run: ./.github/dependencies.sh - - - name: Install Python dependencies - run: pip install -r requirements.txt && python setup.py - - - name: Download all MBTiles artifacts - uses: actions/download-artifact@v4 - - - name: Zip up the pack - run: | - mkdir -p build/${{ inputs.pack }} - mv ${{ inputs.pack }}-rocks/* build/${{ inputs.pack }}/ - mv ${{ inputs.pack }}-contours/* build/${{ inputs.pack }}/ - mv ${{ inputs.pack }}-water/* build/${{ inputs.pack }}/ - mv ${{ inputs.pack }}-ways/* build/${{ inputs.pack }}/ - mv ${{ inputs.pack }}-context/* build/${{ inputs.pack }}/ - cd build/ - zip ${{ inputs.pack }}.${{ inputs.use_pmtiles && 'pmtiles.' || '' }}zip ${{ inputs.pack }}/* - cd ../ - - - name: Make the manifest - run: | - python packs.py manifest \ - --s3-bucket-url https://static.underfoot.rocks \ - --manifest-url https://static.underfoot.rocks/manifest.json - - - name: Set up s3cmd for Digital Ocean - uses: s3-actions/s3cmd@v1.2.0 - with: - provider: digitalocean - region: sfo2 - access_key: ${{ secrets.DO_SPACES_KEY }} - secret_key: ${{ secrets.DO_SPACES_SECRET }} - - - name: Upload pack to Digital Ocean - run: | - s3cmd put build/manifest.json s3://underfoot2/manifest.json - s3cmd setacl s3://underfoot2/manifest.json --acl-public - s3cmd put build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip s3://underfoot2/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip - s3cmd setacl s3://underfoot2/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip --acl-public - - - name: Set up doctl - uses: digitalocean/action-doctl@v2 - with: - token: ${{ secrets.DO_ACCESS_TOKEN }} - - - name: Flush the CDN cache - run: doctl compute cdn flush ee6a9baf-c5bd-423d-bbcf-305e3a3f5e40 + # osm: + # name: Build ways & context + # runs-on: ubuntu-22.04 + # services: + # postgres: + # image: postgis/postgis:12-2.5 + # env: + # POSTGRES_USER: underfoot + # POSTGRES_PASSWORD: underfoot + # POSTGRES_DB: underfoot + # ports: + # - 5432:5432 + # options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + # steps: + # - uses: actions/checkout@v4 + + # - name: Install Ubuntu dependencies + # run: ./.github/dependencies.sh + + # - name: Install Python dependencies + # run: pip install -r requirements.txt && python setup.py + + # - name: Get current year week + # id: calc-year-week + # run: echo "year-week=$(date +'%Y-%U')" >> $GITHUB_OUTPUT + + # - name: Get OSM URL + # id: get-osm-url + # run: UNDERFOOT_PACK=${{ inputs.pack }} && echo "osm-url=$(jq -c -r .osm packs/$UNDERFOOT_PACK.json)" >> $GITHUB_OUTPUT + + # - name: Cache OSM download + # id: cache-osm + # uses: actions/cache@v3 + # with: + # path: "*.osm.pbf" + # key: cache-osm-${{ steps.get-osm-url.outputs.osm-url }}-${{ steps.calc-year-week.outputs.year-week }} + + # - name: Build ways & context + # run: | + # python packs.py ${{ inputs.pack }} --only ways context ${{ inputs.use_pmtiles && '--use-pmtiles' }} + # env: + # PGHOST: 0.0.0.0 + # PGUSER: underfoot + # PGPASSWORD: underfoot + + # - name: Upload ways + # uses: actions/upload-artifact@v4 + # with: + # name: ${{ inputs.pack }}-ways + # path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" + + # - name: Upload context + # uses: actions/upload-artifact@v4 + # with: + # name: ${{ inputs.pack }}-context + # path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" + + # upload-pack: + # name: Archive and upload pack + # runs-on: ubuntu-22.04 + # needs: + # - rocks + # - water + # - contours + # - osm + # steps: + # - uses: actions/checkout@v4 + + # - name: Install Ubuntu dependencies + # run: ./.github/dependencies.sh + + # - name: Install Python dependencies + # run: pip install -r requirements.txt && python setup.py + + # - name: Download all MBTiles artifacts + # uses: actions/download-artifact@v4 + + # - name: Zip up the pack + # run: | + # mkdir -p build/${{ inputs.pack }} + # mv ${{ inputs.pack }}-rocks/* build/${{ inputs.pack }}/ + # mv ${{ inputs.pack }}-contours/* build/${{ inputs.pack }}/ + # mv ${{ inputs.pack }}-water/* build/${{ inputs.pack }}/ + # mv ${{ inputs.pack }}-ways/* build/${{ inputs.pack }}/ + # mv ${{ inputs.pack }}-context/* build/${{ inputs.pack }}/ + # cd build/ + # zip ${{ inputs.pack }}.${{ inputs.use_pmtiles && 'pmtiles.' || '' }}zip ${{ inputs.pack }}/* + # cd ../ + + # - name: Make the manifest + # run: | + # python packs.py manifest \ + # --s3-bucket-url https://static.underfoot.rocks \ + # --manifest-url https://static.underfoot.rocks/manifest.json + + # - name: Set up s3cmd for Digital Ocean + # uses: s3-actions/s3cmd@v1.2.0 + # with: + # provider: digitalocean + # region: sfo2 + # access_key: ${{ secrets.DO_SPACES_KEY }} + # secret_key: ${{ secrets.DO_SPACES_SECRET }} + + # - name: Upload pack to Digital Ocean + # run: | + # s3cmd put build/manifest.json s3://underfoot2/manifest.json + # s3cmd setacl s3://underfoot2/manifest.json --acl-public + # s3cmd put build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip s3://underfoot2/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip + # s3cmd setacl s3://underfoot2/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip --acl-public + + # - name: Set up doctl + # uses: digitalocean/action-doctl@v2 + # with: + # token: ${{ secrets.DO_ACCESS_TOKEN }} + + # - name: Flush the CDN cache + # run: doctl compute cdn flush ee6a9baf-c5bd-423d-bbcf-305e3a3f5e40 diff --git a/elevation.py b/elevation.py index 037fcf3..5429690 100644 --- a/elevation.py +++ b/elevation.py @@ -22,7 +22,8 @@ TABLE_NAME = "contours" -CACHE_DIR = tempfile.mkdtemp(suffix="-elevation-tiles") +# CACHE_DIR = tempfile.mkdtemp(suffix="-elevation-tiles") +CACHE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "elevation-tiles") # pylint: disable=invalid-name @@ -72,8 +73,11 @@ async def cache_tile(tile, client, clean=False, max_retries=3, debug=False): """Caches a tile""" tile_path = f"{tile.z}/{tile.x}/{tile.y}.tif" url = f"https://s3.amazonaws.com/elevation-tiles-prod/geotiff/{tile_path}" + util.log(f"tile url: {url}") file_path = tile_file_path(tile.x, tile.y, tile.z) + util.log(f"file_path: {file_path}") dir_path = os.path.dirname(file_path) + util.log(f"dir_path: {dir_path}") os.makedirs(dir_path, exist_ok=True) if os.path.exists(file_path): util.log(f"cache_tile, path exists: {file_path}") @@ -116,7 +120,7 @@ async def cache_tiles(tiles, clean=False): """Cache multiple tiles""" async with httpx.AsyncClient() as client: # using as_completed with tqdm (https://stackoverflow.com/a/37901797) - tasks = [cache_tile(tile, client, clean=clean) for tile in tiles] + tasks = [cache_tile(tile, client, clean=clean, debug=True) for tile in tiles] pbar = tqdm( asyncio.as_completed(tasks), total=len(tasks), @@ -131,6 +135,8 @@ def make_contours_for_tile(tile, clean=False): """Make contours for a file""" # print("Making contours for {}".format(tile)) tile_path = tile_file_path(tile.x, tile.y, tile.z) + dir_path = os.path.dirname(tile_path) + util.log(f"os.path.exists({dir_path}): {os.path.exists(dir_path)}") if not os.path.exists(tile_path): raise FileNotFoundError(f"Tile file does not exist at {tile_path}") merge_contours_path = tile_file_path(