Lazy primitive group deserialization (#309)

NOTICE.md

@@ -20,7 +20,7 @@ The `planetiler-core` module includes the following software:
   - org.xerial:sqlite-jdbc (Apache license)
   - com.ibm.icu:icu4j ([ICU license](https://github.com/unicode-org/icu/blob/main/icu4c/LICENSE))
   - com.google.guava:guava (Apache license)
-  - org.openstreetmap.osmosis:osmosis-osm-binary (LGPL 3.0)
+  - com.google.protobuf:protobuf-java (BSD 3-Clause License)
   - com.carrotsearch:hppc (Apache license)
   - com.github.jnr:jnr-ffi (Apache license)
   - org.roaringbitmap:RoaringBitmap (Apache license)
@@ -37,6 +37,8 @@ The `planetiler-core` module includes the following software:
   - `ArrayLongMinHeap` implementations from [graphhopper](https://github.com/graphhopper/graphhopper) (Apache license)
   - `Hilbert` implementation
     from [github.com/rawrunprotected/hilbert_curves](https://github.com/rawrunprotected/hilbert_curves) (Public Domain)
+  - `osmformat.proto` and `fileformat.proto` (generates `Osmformat.java` and `Fileformat.java`)
+    from [openstreetmap/OSM-binary](https://github.com/openstreetmap/OSM-binary/tree/master/osmpbf) (MIT License)
 
 Additionally, the `planetiler-basemap` module is based on [OpenMapTiles](https://github.com/openmaptiles/openmaptiles):
 

planetiler-core/pom.xml

@@ -33,9 +33,9 @@
       <version>0.9.30</version>
     </dependency>
     <dependency>
-      <groupId>org.openstreetmap.osmosis</groupId>
-      <artifactId>osmosis-osm-binary</artifactId>
-      <version>0.48.3</version>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <version>3.21.3</version>
     </dependency>
     <dependency>
       <groupId>com.github.jnr</groupId>

planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmInputFile.java

@@ -5,6 +5,7 @@
 import com.onthegomap.planetiler.util.ByteBufferUtil;
 import com.onthegomap.planetiler.util.DiskBacked;
 import com.onthegomap.planetiler.util.FileUtils;
+import crosby.binary.Fileformat.BlobHeader;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.ByteBuffer;
@@ -15,7 +16,6 @@
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 import org.locationtech.jts.geom.Envelope;
-import org.openstreetmap.osmosis.osmbinary.Fileformat.BlobHeader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmReader.java

@@ -5,7 +5,6 @@
 
 import com.carrotsearch.hppc.IntObjectHashMap;
 import com.carrotsearch.hppc.LongArrayList;
-import com.carrotsearch.hppc.LongHashSet;
 import com.carrotsearch.hppc.LongObjectHashMap;
 import com.carrotsearch.hppc.ObjectIntHashMap;
 import com.onthegomap.planetiler.FeatureCollector;
@@ -53,6 +52,7 @@
 import org.locationtech.jts.geom.Polygon;
 import org.locationtech.jts.geom.impl.CoordinateArraySequence;
 import org.locationtech.jts.geom.impl.PackedCoordinateSequence;
+import org.roaringbitmap.longlong.Roaring64Bitmap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -86,7 +86,7 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate {
   private final Object wayToRelationsLock = new Object();
   // for multipolygons need to store way info (20m ways, 800m nodes) to use when processing relations (4.5m)
   // ~300mb
-  private LongHashSet waysInMultipolygon = new LongHashSet();
+  private Roaring64Bitmap waysInMultipolygon = new Roaring64Bitmap();
   private final Object waysInMultipolygonLock = new Object();
   // ~7GB
   private LongLongMultimap.Replaceable multipolygonWayGeometries;
@@ -541,7 +541,7 @@ private List<RelationMember<OsmRelationInfo>> getRelationMembershipForWay(long w
   @Override
   public long estimateMemoryUsageBytes() {
     long size = 0;
-    size += estimateSize(waysInMultipolygon);
+    size += waysInMultipolygon.serializedSizeInBytes();
     // multipolygonWayGeometries is reported separately
     size += estimateSize(wayToRelations);
     size += estimateSize(relationInfo);

planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java

@@ -4,7 +4,10 @@
 
 import com.carrotsearch.hppc.LongArrayList;
 import com.google.common.collect.Iterators;
+import com.google.protobuf.InvalidProtocolBufferException;
 import com.onthegomap.planetiler.reader.FileFormatException;
+import crosby.binary.Fileformat;
+import crosby.binary.Osmformat;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.ByteBuffer;
@@ -20,8 +23,6 @@
 import java.util.zip.DataFormatException;
 import java.util.zip.Inflater;
 import org.locationtech.jts.geom.Envelope;
-import org.openstreetmap.osmosis.osmbinary.Fileformat;
-import org.openstreetmap.osmosis.osmbinary.Osmformat;
 
 /**
  * Converts PBF block data into decoded entities. This class was adapted from Osmosis to expose an iterator over blocks
@@ -127,12 +128,7 @@ public static OsmHeader decodeHeader(byte[] raw) {
 
   @Override
   public Iterator<OsmElement> iterator() {
-    return Iterators.concat(block.getPrimitivegroupList().stream().map(primitiveGroup -> Iterators.concat(
-      new DenseNodeIterator(primitiveGroup.getDense()),
-      new NodeIterator(primitiveGroup.getNodesList()),
-      new WayIterator(primitiveGroup.getWaysList()),
-      new RelationIterator(primitiveGroup.getRelationsList())
-    )).iterator());
+    return Iterators.concat(new PrimitiveGroupIterator());
   }
 
   private Map<String, Object> buildTags(int num, IntUnaryOperator key, IntUnaryOperator value) {
@@ -148,6 +144,43 @@ private Map<String, Object> buildTags(int num, IntUnaryOperator key, IntUnaryOpe
     return Collections.emptyMap();
   }
 
+  private OsmElement.Info parseInfo(Osmformat.Info info) {
+    return info == null ? null : new OsmElement.Info(
+      info.getChangeset(),
+      info.getTimestamp(),
+      info.getUid(),
+      info.getVersion(),
+      fieldDecoder.decodeString(info.getUserSid())
+    );
+  }
+
+  private class PrimitiveGroupIterator implements Iterator<Iterator<OsmElement>> {
+    private int i = 0;
+
+    @Override
+    public boolean hasNext() {
+      return i < block.getPrimitivegroupCount();
+    }
+
+    @Override
+    public Iterator<OsmElement> next() {
+      if (!hasNext()) {
+        throw new NoSuchElementException();
+      }
+      try {
+        var primitiveGroup = Osmformat.PrimitiveGroup.parseFrom(block.getPrimitivegroup(i++));
+        return Iterators.concat(
+          new DenseNodeIterator(primitiveGroup.getDense()),
+          new NodeIterator(primitiveGroup.getNodesList()),
+          new WayIterator(primitiveGroup.getWaysList()),
+          new RelationIterator(primitiveGroup.getRelationsList())
+        );
+      } catch (InvalidProtocolBufferException e) {
+        throw new FileFormatException("Unable to parse primitive group", e);
+      }
+    }
+  }
+
   private class NodeIterator implements Iterator<OsmElement.Node> {
 
     private final List<Osmformat.Node> nodes;
@@ -273,16 +306,6 @@ public OsmElement.Way next() {
     }
   }
 
-  private OsmElement.Info parseInfo(Osmformat.Info info) {
-    return info == null ? null : new OsmElement.Info(
-      info.getChangeset(),
-      info.getTimestamp(),
-      info.getUid(),
-      info.getVersion(),
-      fieldDecoder.decodeString(info.getUserSid())
-    );
-  }
-
   private class DenseNodeIterator implements Iterator<OsmElement.Node> {
 
     final Osmformat.DenseNodes nodes;

planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfFieldDecoder.java

@@ -2,8 +2,8 @@
 // See NOTICE.md here or copying.txt from https://github.com/openstreetmap/osmosis/blob/master/package/copying.txt for details.
 package com.onthegomap.planetiler.reader.osm;
 
+import crosby.binary.Osmformat;
 import java.util.Date;
-import org.openstreetmap.osmosis.osmbinary.Osmformat;
 
 /**
  * Manages decoding of the lower level PBF data structures.