From 187dc81ef97a915faa5704fe300f644a5bb6ef4d Mon Sep 17 00:00:00 2001 From: Ken-ichi Ueda Date: Wed, 11 Sep 2024 14:09:49 -0700 Subject: [PATCH] Remove logging, clean up --- .github/workflows/build-pack.yml | 408 ++++++++++---------- elevation.py | 17 +- packs/us-ca-northwest.json | 2 +- sources/gam_23_weed_1963_gis/citation.json | 2 +- sources/gam_24_ukiah_1960_gis/citation.json | 2 +- 5 files changed, 209 insertions(+), 222 deletions(-) diff --git a/.github/workflows/build-pack.yml b/.github/workflows/build-pack.yml index 98107af..902aa59 100644 --- a/.github/workflows/build-pack.yml +++ b/.github/workflows/build-pack.yml @@ -20,94 +20,51 @@ on: required: true jobs: - # rocks: - # name: Build rocks - # runs-on: ubuntu-22.04 - # services: - # postgres: - # image: postgis/postgis:12-2.5 - # env: - # POSTGRES_USER: underfoot - # POSTGRES_PASSWORD: underfoot - # POSTGRES_DB: underfoot - # ports: - # - 5432:5432 - # options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - # steps: - # - uses: actions/checkout@v4 - - # - name: Install Ubuntu dependencies - # run: ./.github/dependencies.sh - - # - name: Install Python dependencies - # run: pip install -r requirements.txt && python setup.py - - # - id: cache-source-work-rocks - # name: Cache source work for rocks - # uses: actions/cache@v3 - # with: - # path: sources/work-* - # key: cache-source-work-rocks-2-${{ inputs.pack }}-${{ hashFiles(format('packs/{0}.json', inputs.pack)) }} - - # - name: Build rocks - # run: | - # python packs.py ${{ inputs.pack }} --only rocks ${{ inputs.use_pmtiles && '--use-pmtiles' }} - # env: - # PGHOST: 0.0.0.0 - # PGUSER: underfoot - # PGPASSWORD: underfoot - - # - name: Upload rocks - # uses: actions/upload-artifact@v4 - # with: - # name: ${{ inputs.pack }}-rocks - # path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" - - # water: - # name: Build water - # runs-on: ubuntu-22.04 - # services: - # postgres: - # image: postgis/postgis:12-2.5 - # env: - # POSTGRES_USER: underfoot - # POSTGRES_PASSWORD: underfoot - # POSTGRES_DB: underfoot - # ports: - # - 5432:5432 - # options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - # steps: - # - uses: actions/checkout@v4 - - # - name: Install Ubuntu dependencies - # run: ./.github/dependencies.sh - - # - name: Install Python dependencies - # run: pip install -r requirements.txt && python setup.py - - # - id: cache-source-work-water - # name: Cache source work for water - # uses: actions/cache@v3 - # with: - # path: sources/work-* - # key: cache-source-work-water-${{ inputs.pack }}-${{ hashFiles(format('packs/{0}.json', inputs.pack)) }} - - # - name: Build water - # run: | - # python packs.py ${{ inputs.pack }} --only water ${{ inputs.use_pmtiles && '--use-pmtiles' }} - # env: - # PGHOST: 0.0.0.0 - # PGUSER: underfoot - # PGPASSWORD: underfoot - - # - name: Upload water - # uses: actions/upload-artifact@v4 - # with: - # name: ${{ inputs.pack }}-water - # path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" + rocks: + name: Build rocks + runs-on: ubuntu-22.04 + services: + postgres: + image: postgis/postgis:12-2.5 + env: + POSTGRES_USER: underfoot + POSTGRES_PASSWORD: underfoot + POSTGRES_DB: underfoot + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + steps: + - uses: actions/checkout@v4 - contours: - name: Build contours + - name: Install Ubuntu dependencies + run: ./.github/dependencies.sh + + - name: Install Python dependencies + run: pip install -r requirements.txt && python setup.py + + - id: cache-source-work-rocks + name: Cache source work for rocks + uses: actions/cache@v3 + with: + path: sources/work-* + key: cache-source-work-rocks-2-${{ inputs.pack }}-${{ hashFiles(format('packs/{0}.json', inputs.pack)) }} + + - name: Build rocks + run: | + python packs.py ${{ inputs.pack }} --only rocks ${{ inputs.use_pmtiles && '--use-pmtiles' }} + env: + PGHOST: 0.0.0.0 + PGUSER: underfoot + PGPASSWORD: underfoot + + - name: Upload rocks + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.pack }}-rocks + path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" + + water: + name: Build water runs-on: ubuntu-22.04 services: postgres: @@ -120,10 +77,49 @@ jobs: - 5432:5432 options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - - name: Check tile URLs work + - uses: actions/checkout@v4 + + - name: Install Ubuntu dependencies + run: ./.github/dependencies.sh + + - name: Install Python dependencies + run: pip install -r requirements.txt && python setup.py + + - id: cache-source-work-water + name: Cache source work for water + uses: actions/cache@v3 + with: + path: sources/work-* + key: cache-source-work-water-${{ inputs.pack }}-${{ hashFiles(format('packs/{0}.json', inputs.pack)) }} + + - name: Build water run: | - curl -v -f --fail-early --connect-timeout 30 --max-time 30 -o "/tmp/test.tif" "https://s3.amazonaws.com/elevation-tiles-prod/geotiff/12/646/1537.tif" + python packs.py ${{ inputs.pack }} --only water ${{ inputs.use_pmtiles && '--use-pmtiles' }} + env: + PGHOST: 0.0.0.0 + PGUSER: underfoot + PGPASSWORD: underfoot + + - name: Upload water + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.pack }}-water + path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" + contours: + name: Build contours + runs-on: ubuntu-22.04 + services: + postgres: + image: postgis/postgis:12-2.5 + env: + POSTGRES_USER: underfoot + POSTGRES_PASSWORD: underfoot + POSTGRES_DB: underfoot + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + steps: - uses: actions/checkout@v4 - name: Install Ubuntu dependencies @@ -153,120 +149,120 @@ jobs: name: ${{ inputs.pack }}-contours path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" - # osm: - # name: Build ways & context - # runs-on: ubuntu-22.04 - # services: - # postgres: - # image: postgis/postgis:12-2.5 - # env: - # POSTGRES_USER: underfoot - # POSTGRES_PASSWORD: underfoot - # POSTGRES_DB: underfoot - # ports: - # - 5432:5432 - # options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - # steps: - # - uses: actions/checkout@v4 - - # - name: Install Ubuntu dependencies - # run: ./.github/dependencies.sh - - # - name: Install Python dependencies - # run: pip install -r requirements.txt && python setup.py - - # - name: Get current year week - # id: calc-year-week - # run: echo "year-week=$(date +'%Y-%U')" >> $GITHUB_OUTPUT - - # - name: Get OSM URL - # id: get-osm-url - # run: UNDERFOOT_PACK=${{ inputs.pack }} && echo "osm-url=$(jq -c -r .osm packs/$UNDERFOOT_PACK.json)" >> $GITHUB_OUTPUT - - # - name: Cache OSM download - # id: cache-osm - # uses: actions/cache@v3 - # with: - # path: "*.osm.pbf" - # key: cache-osm-${{ steps.get-osm-url.outputs.osm-url }}-${{ steps.calc-year-week.outputs.year-week }} - - # - name: Build ways & context - # run: | - # python packs.py ${{ inputs.pack }} --only ways context ${{ inputs.use_pmtiles && '--use-pmtiles' }} - # env: - # PGHOST: 0.0.0.0 - # PGUSER: underfoot - # PGPASSWORD: underfoot - - # - name: Upload ways - # uses: actions/upload-artifact@v4 - # with: - # name: ${{ inputs.pack }}-ways - # path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" - - # - name: Upload context - # uses: actions/upload-artifact@v4 - # with: - # name: ${{ inputs.pack }}-context - # path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" - - # upload-pack: - # name: Archive and upload pack - # runs-on: ubuntu-22.04 - # needs: - # - rocks - # - water - # - contours - # - osm - # steps: - # - uses: actions/checkout@v4 - - # - name: Install Ubuntu dependencies - # run: ./.github/dependencies.sh - - # - name: Install Python dependencies - # run: pip install -r requirements.txt && python setup.py - - # - name: Download all MBTiles artifacts - # uses: actions/download-artifact@v4 - - # - name: Zip up the pack - # run: | - # mkdir -p build/${{ inputs.pack }} - # mv ${{ inputs.pack }}-rocks/* build/${{ inputs.pack }}/ - # mv ${{ inputs.pack }}-contours/* build/${{ inputs.pack }}/ - # mv ${{ inputs.pack }}-water/* build/${{ inputs.pack }}/ - # mv ${{ inputs.pack }}-ways/* build/${{ inputs.pack }}/ - # mv ${{ inputs.pack }}-context/* build/${{ inputs.pack }}/ - # cd build/ - # zip ${{ inputs.pack }}.${{ inputs.use_pmtiles && 'pmtiles.' || '' }}zip ${{ inputs.pack }}/* - # cd ../ - - # - name: Make the manifest - # run: | - # python packs.py manifest \ - # --s3-bucket-url https://static.underfoot.rocks \ - # --manifest-url https://static.underfoot.rocks/manifest.json - - # - name: Set up s3cmd for Digital Ocean - # uses: s3-actions/s3cmd@v1.2.0 - # with: - # provider: digitalocean - # region: sfo2 - # access_key: ${{ secrets.DO_SPACES_KEY }} - # secret_key: ${{ secrets.DO_SPACES_SECRET }} - - # - name: Upload pack to Digital Ocean - # run: | - # s3cmd put build/manifest.json s3://underfoot2/manifest.json - # s3cmd setacl s3://underfoot2/manifest.json --acl-public - # s3cmd put build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip s3://underfoot2/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip - # s3cmd setacl s3://underfoot2/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip --acl-public - - # - name: Set up doctl - # uses: digitalocean/action-doctl@v2 - # with: - # token: ${{ secrets.DO_ACCESS_TOKEN }} - - # - name: Flush the CDN cache - # run: doctl compute cdn flush ee6a9baf-c5bd-423d-bbcf-305e3a3f5e40 + osm: + name: Build ways & context + runs-on: ubuntu-22.04 + services: + postgres: + image: postgis/postgis:12-2.5 + env: + POSTGRES_USER: underfoot + POSTGRES_PASSWORD: underfoot + POSTGRES_DB: underfoot + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + steps: + - uses: actions/checkout@v4 + + - name: Install Ubuntu dependencies + run: ./.github/dependencies.sh + + - name: Install Python dependencies + run: pip install -r requirements.txt && python setup.py + + - name: Get current year week + id: calc-year-week + run: echo "year-week=$(date +'%Y-%U')" >> $GITHUB_OUTPUT + + - name: Get OSM URL + id: get-osm-url + run: UNDERFOOT_PACK=${{ inputs.pack }} && echo "osm-url=$(jq -c -r .osm packs/$UNDERFOOT_PACK.json)" >> $GITHUB_OUTPUT + + - name: Cache OSM download + id: cache-osm + uses: actions/cache@v3 + with: + path: "*.osm.pbf" + key: cache-osm-${{ steps.get-osm-url.outputs.osm-url }}-${{ steps.calc-year-week.outputs.year-week }} + + - name: Build ways & context + run: | + python packs.py ${{ inputs.pack }} --only ways context ${{ inputs.use_pmtiles && '--use-pmtiles' }} + env: + PGHOST: 0.0.0.0 + PGUSER: underfoot + PGPASSWORD: underfoot + + - name: Upload ways + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.pack }}-ways + path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" + + - name: Upload context + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.pack }}-context + path: "build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}/" + + upload-pack: + name: Archive and upload pack + runs-on: ubuntu-22.04 + needs: + - rocks + - water + - contours + - osm + steps: + - uses: actions/checkout@v4 + + - name: Install Ubuntu dependencies + run: ./.github/dependencies.sh + + - name: Install Python dependencies + run: pip install -r requirements.txt && python setup.py + + - name: Download all MBTiles artifacts + uses: actions/download-artifact@v4 + + - name: Zip up the pack + run: | + mkdir -p build/${{ inputs.pack }} + mv ${{ inputs.pack }}-rocks/* build/${{ inputs.pack }}/ + mv ${{ inputs.pack }}-contours/* build/${{ inputs.pack }}/ + mv ${{ inputs.pack }}-water/* build/${{ inputs.pack }}/ + mv ${{ inputs.pack }}-ways/* build/${{ inputs.pack }}/ + mv ${{ inputs.pack }}-context/* build/${{ inputs.pack }}/ + cd build/ + zip ${{ inputs.pack }}.${{ inputs.use_pmtiles && 'pmtiles.' || '' }}zip ${{ inputs.pack }}/* + cd ../ + + - name: Make the manifest + run: | + python packs.py manifest \ + --s3-bucket-url https://static.underfoot.rocks \ + --manifest-url https://static.underfoot.rocks/manifest.json + + - name: Set up s3cmd for Digital Ocean + uses: s3-actions/s3cmd@v1.2.0 + with: + provider: digitalocean + region: sfo2 + access_key: ${{ secrets.DO_SPACES_KEY }} + secret_key: ${{ secrets.DO_SPACES_SECRET }} + + - name: Upload pack to Digital Ocean + run: | + s3cmd put build/manifest.json s3://underfoot2/manifest.json + s3cmd setacl s3://underfoot2/manifest.json --acl-public + s3cmd put build/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip s3://underfoot2/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip + s3cmd setacl s3://underfoot2/${{ inputs.pack }}${{ inputs.use_pmtiles && '.pmtiles' || '' }}.zip --acl-public + + - name: Set up doctl + uses: digitalocean/action-doctl@v2 + with: + token: ${{ secrets.DO_ACCESS_TOKEN }} + + - name: Flush the CDN cache + run: doctl compute cdn flush ee6a9baf-c5bd-423d-bbcf-305e3a3f5e40 diff --git a/elevation.py b/elevation.py index 383dd58..24983f9 100644 --- a/elevation.py +++ b/elevation.py @@ -25,7 +25,6 @@ TABLE_NAME = "contours" -# CACHE_DIR = tempfile.mkdtemp(suffix="-elevation-tiles") CACHE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "elevation-tiles") @@ -82,14 +81,12 @@ async def cache_tile(tile, client, clean=False, max_retries=3, debug=False): """Caches a tile""" tile_path = f"{tile.z}/{tile.x}/{tile.y}.tif" url = f"https://s3.amazonaws.com/elevation-tiles-prod/geotiff/{tile_path}" - # util.log(f"tile url: {url}") file_path = tile_file_path(tile.x, tile.y, tile.z) - # util.log(f"file_path: {file_path}") dir_path = os.path.dirname(file_path) - # util.log(f"dir_path: {dir_path}") os.makedirs(dir_path, exist_ok=True) if os.path.exists(file_path): - util.log(f"cache_tile, path exists: {file_path}") + if debug: + util.log(f"cache_tile, path exists: {file_path}") if clean: os.remove(file_path) else: @@ -97,21 +94,17 @@ async def cache_tile(tile, client, clean=False, max_retries=3, debug=False): # TODO handle errors, client abort, server abort async for try_num in async_range(1, max_retries + 1): try: - # if debug: - # util.log(f"getting {url}") + if debug: + util.log(f"getting {url}") download = await client.get(url) - # util.log(f"download.status_code: {download.status_code}") if download.status_code != 200: util.log( f"Request for {url} failed with {download.status_code}, " "skipping..." ) return - util.log(f"opening tile path {file_path}") async with aiofiles.open(file_path, 'wb') as outfile: - util.log(f"writing bytes to {file_path}") async for chunk in download.aiter_bytes(): - util.log(f"writing chunk to {file_path}") await outfile.write(chunk) break except ( @@ -163,10 +156,8 @@ async def cache_tiles(tiles, clean=False): def make_contours_for_tile(tile, clean=False): """Make contours for a file""" - # print("Making contours for {}".format(tile)) tile_path = tile_file_path(tile.x, tile.y, tile.z) dir_path = os.path.dirname(tile_path) - util.log(f"os.path.exists({dir_path}): {os.path.exists(dir_path)}") if not os.path.exists(tile_path): raise FileNotFoundError(f"Tile file does not exist at {tile_path}") merge_contours_path = tile_file_path( diff --git a/packs/us-ca-northwest.json b/packs/us-ca-northwest.json index 30fbd0c..bdcc2df 100644 --- a/packs/us-ca-northwest.json +++ b/packs/us-ca-northwest.json @@ -5,7 +5,7 @@ "bottom": 38.950663670055704, "left": -124.56869985094606 , "right": -121.97320529294143 , - "top": 42.06463966550385 + "top": 42.06463966550385 }, "geojson": { "$ref": "file://./us-ca-northwest.geojson" diff --git a/sources/gam_23_weed_1963_gis/citation.json b/sources/gam_23_weed_1963_gis/citation.json index 18cce09..9ddcf06 100644 --- a/sources/gam_23_weed_1963_gis/citation.json +++ b/sources/gam_23_weed_1963_gis/citation.json @@ -20,4 +20,4 @@ ] } } -] \ No newline at end of file +] diff --git a/sources/gam_24_ukiah_1960_gis/citation.json b/sources/gam_24_ukiah_1960_gis/citation.json index 7b65e51..7892cf5 100644 --- a/sources/gam_24_ukiah_1960_gis/citation.json +++ b/sources/gam_24_ukiah_1960_gis/citation.json @@ -24,4 +24,4 @@ ] } } -] \ No newline at end of file +]