Move basemap profile into OpenMapTiles submodule (#258)

onthegomap · Jul 29, 2022 · 88c7f56 · 88c7f56
1 parent 232834e
commit 88c7f56
Show file tree

Hide file tree

Showing 75 changed files with 352 additions and 14,329 deletions.
diff --git a/.github/cache-sources-action/action.yml b/.github/cache-sources-action/action.yml
@@ -12,7 +12,7 @@ runs:
     - name: Get Date
       id: get-data
       run: |
-        echo "::set-output name=hash::${{ hashFiles('**/Planetiler.java', '**/Downloader.java', '**/Geofabrik.java', '**/BasemapMain.java') }}"
+        echo "::set-output name=hash::${{ hashFiles('**/Planetiler.java', '**/Downloader.java', '**/Geofabrik.java', '**/OpenMapTilesMain.java') }}"
         echo "::set-output name=date::$(date -u "+%Y-%m-%d")"
       shell: bash
       working-directory: ${{ inputs.basedir }}

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -13,6 +13,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+        with:
+          submodules: true
       - name: Run link check
         uses: gaurav-nelson/[email protected]
         with:

diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
@@ -16,6 +16,8 @@ jobs:
     timeout-minutes: 15
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: true
       - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
@@ -45,6 +47,8 @@ jobs:
     timeout-minutes: 15
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: true
       - name: Set up JDK ${{ matrix.jdk }}
         uses: actions/setup-java@v3
         with:
@@ -66,6 +70,8 @@ jobs:
     timeout-minutes: 15
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: true
       - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
@@ -74,8 +80,8 @@ jobs:
           cache: 'maven'
       - run: ./scripts/regenerate-openmaptiles.sh
       # Skip spotless since that gets checked in a separate task
-      - run: ./mvnw -Dspotless.check.skip  -DskipTests --batch-mode -no-transfer-progress clean install -pl planetiler-basemap -am
-      - run: ./mvnw -Dspotless.check.skip --batch-mode -no-transfer-progress verify -pl planetiler-basemap
+      - run: ./mvnw -Dspotless.check.skip  -DskipTests --batch-mode -no-transfer-progress clean install -pl planetiler-openmaptiles -am
+      - run: ./mvnw -Dspotless.check.skip --batch-mode -no-transfer-progress verify -pl planetiler-openmaptiles
 
   examples:
     name: Example project
@@ -84,6 +90,8 @@ jobs:
     continue-on-error: true
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: true
       - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
@@ -108,6 +116,8 @@ jobs:
     timeout-minutes: 15
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: true
       - name: Cache data/sources
         uses: ./.github/cache-sources-action
       - name: Set up JDK

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
@@ -30,11 +30,13 @@ jobs:
         uses: actions/checkout@v3
         with:
           path: branch
+          submodules: true
       - name: 'Checkout base'
         uses: actions/checkout@v3
         with:
           path: base
           ref: ${{ github.event.pull_request.base.sha }}
+          submodules: true
       - name: Cache data/sources
         uses: ./branch/.github/cache-sources-action
         with:

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -26,6 +26,8 @@ jobs:
             version = context.payload.inputs.version;
             if (/^v/.test(version)) throw new Error("Bad version number: " + version)
       - uses: actions/checkout@v3
+        with:
+          submodules: true
       - name: Cache data/sources
         uses: ./.github/cache-sources-action
       - uses: actions/setup-java@v3

diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml
@@ -22,6 +22,8 @@ jobs:
       packages: write
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: true
       - name: Cache data/sources
         uses: ./.github/cache-sources-action
       - name: Set up JDK

diff --git a/.github/workflows/sonar.yml b/.github/workflows/sonar.yml
@@ -19,6 +19,7 @@ jobs:
         with:
           # Disabling shallow clone is recommended for improving relevancy of reporting
           fetch-depth: 0
+          submodules: true
       - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:

diff --git a/.github/workflows/update-pr.yml b/.github/workflows/update-pr.yml
@@ -19,6 +19,8 @@ jobs:
       # report status back to pull request
       - uses: haya14busa/action-workflow_run-status@v1
       - uses: actions/checkout@v3
+        with:
+          submodules: true
       - name: 'Download branch build info'
         uses: dawidd6/action-download-artifact@v2
         with:

diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,4 @@
+[submodule "planetiler-openmaptiles"]
+	path = planetiler-openmaptiles
+	url = https://github.com/openmaptiles/planetiler-openmaptiles.git
+	branch = main
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -10,7 +10,7 @@ Pull requests are welcome! Any pull request should:
 
 To set up your local development environment:
 
-- Fork the repo
+- Fork the repo [setup submodules](README.md#git-submodules)
 - Install Java 16 or later. You can download Java manually from [Adoptium](https://adoptium.net/installation.html) or
   use:
   - [Windows installer](https://adoptium.net/installation.html#windows-msi)

diff --git a/NOTICE.md b/NOTICE.md
@@ -39,9 +39,6 @@ The `planetiler-core` module includes the following software:
     from [github.com/rawrunprotected/hilbert_curves](https://github.com/rawrunprotected/hilbert_curves) (Public Domain)
   - `osmformat.proto` and `fileformat.proto` (generates `Osmformat.java` and `Fileformat.java`)
     from [openstreetmap/OSM-binary](https://github.com/openstreetmap/OSM-binary/tree/master/osmpbf) (MIT License)
-
-Additionally, the `planetiler-basemap` module is based on [OpenMapTiles](https://github.com/openmaptiles/openmaptiles):
-
 - Maven Dependencies:
   - org.yaml:snakeyaml (Apache license)
   - org.commonmark:commonmark (BSD 2-clause license)

diff --git a/PLANET.md b/PLANET.md
@@ -1,6 +1,7 @@
 # Generating a Map of the World
 
-To generate a map of the world using the built-in [basemap profile](planetiler-basemap), you will need a machine with
+To generate a map of the world using the built-in [OpenMapTiles profile](https://github.com/openmaptiles/planetiler-openmaptiles), you will need a
+machine with
 Java 16 or later installed and at least 10x as much disk space and at least 0.5x as much RAM as the `planet.osm.pbf`
 file you start from. All testing has been done using Digital Ocean droplets with dedicated
 vCPUs ([referral link](https://m.do.co/c/a947e99aab25)) and OpenJDK 17 installed through `apt`. Planetiler splits work
@@ -66,7 +67,7 @@ java -Xmx20g \
 
 Run with `--help` to see all available arguments.
 
-NOTE: The default basemap profile merges nearby buildings at zoom-level 13 (for example,
+NOTE: The default OpenMapTiles profile merges nearby buildings at zoom-level 13 (for example,
 see [Boston](https://onthegomap.github.io/planetiler-demo/#13.08/42.35474/-71.06597)). This adds about 14 CPU hours (~50
 minutes with 16 CPUs) to planet generation time and can be disabled using `--building-merge-z13=false`.
 

diff --git a/README.md b/README.md
@@ -25,12 +25,12 @@ See the [live demo](https://onthegomap.github.io/planetiler-demo/) of vector til
 the [OpenStreetMap Americana Project](https://github.com/ZeLonewolf/openstreetmap-americana/).
 
 [![Planetiler Demo Screenshot](./diagrams/demo.png)](https://onthegomap.github.io/planetiler-demo/)
-Style [© OpenMapTiles](https://www.openmaptiles.org/)
-&middot; Data [© OpenStreetMap contributors](https://www.openstreetmap.org/copyright)
+[© OpenMapTiles](https://www.openmaptiles.org/) [© OpenStreetMap contributors](https://www.openstreetmap.org/copyright)
 
 ## Usage
 
-To generate a map of an area using the [basemap profile](planetiler-basemap), you will need:
+To generate a map of an area using the [OpenMapTiles profile](https://github.com/openmaptiles/planetiler-openmaptiles),
+you will need:
 
 - Java 16+ (see [CONTRIBUTING.md](CONTRIBUTING.md)) or [Docker](https://docs.docker.com/get-docker/)
 - at least 1GB of free disk space plus 5-10x the size of the `.osm.pbf` file
@@ -53,7 +53,8 @@ Or using Docker:
 docker run -e JAVA_TOOL_OPTIONS="-Xmx1g" -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:latest --download --area=monaco
 ```
 
-:warning: This starts off by downloading about 1GB of [data sources](NOTICE.md#data) required by the basemap profile
+:warning: This starts off by downloading about 1GB of [data sources](NOTICE.md#data) required by the OpenMapTiles
+profile
 including ~750MB for [ocean polygons](https://osmdata.openstreetmap.de/data/water-polygons.html) and ~240MB
 for [Natural Earth Data](https://www.naturalearthdata.com/).
 
@@ -107,6 +108,32 @@ Some common arguments:
 - `--force` overwrites the output file
 - `--help` shows all of the options and exits
 
+### Git submodules
+
+Planetiler has a submodule dependency
+on [planetiler-openmaptiles](https://github.com/openmaptiles/planetiler-openmaptiles). Add `--recurse-submodules`
+to `git clone`, `git pull`, or `git checkout` commands to also update submodule dependencies.
+
+To clone the repo with submodules:
+
+```bash
+git clone --recurse-submodules https://github.com/onthegomap/planetiler.git
+```
+
+If you already pulled the repo, you can initialize submodules with:
+
+```bash
+git submodule update --init
+```
+
+To force git to always update submodules (recommended), run this command in your local repo:
+
+```bash
+git config --local submodule.recurse true
+```
+
+Learn more about working with submodules [here](https://git-scm.com/book/en/v2/Git-Tools-Submodules).
+
 ## Generating a Map of the World
 
 See [PLANET.md](PLANET.md).
@@ -117,7 +144,7 @@ See the [planetiler-examples](planetiler-examples) project.
 
 ## Benchmarks
 
-Some example runtimes for the Basemap OpenMapTiles-compatible profile (excluding downloading resources):
+Some example runtimes for the OpenMapTiles profile (excluding downloading resources):
 
 |                                                                   Input                                                                   | Version |             Machine             |              Time               | mbtiles size |                                                          Logs                                                          |
 |-------------------------------------------------------------------------------------------------------------------------------------------|---------|---------------------------------|---------------------------------|--------------|------------------------------------------------------------------------------------------------------------------------|
@@ -142,8 +169,9 @@ Merging nearby buildings at z13 is very expensive, when run with `--building-mer
 Some other tools that generate vector tiles from OpenStreetMap data:
 
 - [OpenMapTiles](https://github.com/openmaptiles/openmaptiles) is the reference implementation of
-  the [OpenMapTiles schema](https://openmaptiles.org/schema/) that the [basemap profile](planetiler-basemap) is based
-  on. It uses an intermediate postgres database and operates in two modes:
+  the [OpenMapTiles schema](https://openmaptiles.org/schema/) that
+  the [OpenMapTiles profile](https://github.com/openmaptiles/planetiler-openmaptiles)
+  is based on. It uses an intermediate postgres database and operates in two modes:
   1. Import data into database (~1 day) then serve vector tiles directly from the database. Tile serving is slower and
      requires bigger machines, but lets you easily incorporate realtime updates
   2. Import data into database (~1 day) then pregenerate every tile for the planet into an mbtiles file which
@@ -172,7 +200,7 @@ download regularly-updated tilesets.
   using [JTS geometry utilities](https://github.com/locationtech/jts)
 - Merge nearby lines or polygons with the same tags before emitting vector tiles
 - Automatically fixes self-intersecting polygons
-- Built-in basemap profile based on [OpenMapTiles](https://openmaptiles.org/) v3.13.1
+- Built-in OpenMapTiles profile based on [OpenMapTiles](https://openmaptiles.org/) v3.13.1
 - Optionally download additional name translations for elements from Wikidata
 - Export real-time stats to a [prometheus push gateway](https://github.com/prometheus/pushgateway) using
   `--pushgateway=http://user:password@ip` argument (and a [grafana dashboard](grafana.json) for viewing)
@@ -244,7 +272,8 @@ Planetiler is made possible by these awesome open source projects:
 
 - [OpenMapTiles](https://openmaptiles.org/) for the [schema](https://openmaptiles.org/schema/)
   and [reference implementation](https://github.com/openmaptiles/openmaptiles)
-  that the [basemap profile](planetiler-basemap/src/main/java/com/onthegomap/planetiler/basemap/layers)
+  that
+  the [openmaptiles profile](https://github.com/openmaptiles/planetiler-openmaptiles/tree/main/src/main/java/com/onthegomap/planetiler/openmaptiles/layers)
   is based on
 - [Graphhopper](https://www.graphhopper.com/) for basis of utilities to process OpenStreetMap data in Java
 - [JTS Topology Suite](https://github.com/locationtech/jts) for working with vector geometries

diff --git a/config-example.properties b/config-example.properties
@@ -80,7 +80,7 @@
 # mbtiles_version=0.1
 # mbtiles_type=baselayer OR overlay
 
-################# Basemap profile config #################
+################# OpenMapTiles profile config #################
 
 # Set the input file name and automatically determine the https://download.geofabrik.de/ download URL for a region by name:
 # area=monaco

diff --git a/planetiler-basemap/README.md b/planetiler-basemap/README.md
diff --git a/planetiler-basemap/pom.xml b/planetiler-basemap/pom.xml
@@ -1,56 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-
-  <artifactId>planetiler-basemap</artifactId>
-
-  <parent>
-    <groupId>com.onthegomap.planetiler</groupId>
-    <artifactId>planetiler-parent</artifactId>
-    <version>0.5-SNAPSHOT</version>
-  </parent>
-
-  <dependencies>
-    <dependency>
-      <groupId>com.onthegomap.planetiler</groupId>
-      <artifactId>planetiler-core</artifactId>
-      <version>${project.parent.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.yaml</groupId>
-      <artifactId>snakeyaml</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.commonmark</groupId>
-      <artifactId>commonmark</artifactId>
-      <version>0.19.0</version>
-    </dependency>
-
-    <dependency>
-      <groupId>com.onthegomap.planetiler</groupId>
-      <artifactId>planetiler-core</artifactId>
-      <version>${project.parent.version}</version>
-      <type>test-jar</type>
-      <scope>test</scope>
-    </dependency>
-  </dependencies>
-
-  <build>
-    <plugins>
-      <plugin>
-        <groupId>io.github.zlika</groupId>
-        <artifactId>reproducible-build-maven-plugin</artifactId>
-      </plugin>
-
-      <plugin>
-        <artifactId>maven-deploy-plugin</artifactId>
-        <configuration>
-          <!-- we don't want to deploy this module -->
-          <skip>true</skip>
-        </configuration>
-      </plugin>
-    </plugins>
-  </build>
-</project>