diff --git a/scripts/import/mapbox_country_pack.py b/scripts/import/mapbox_country_pack.py index 98478c8e..2eca614e 100644 --- a/scripts/import/mapbox_country_pack.py +++ b/scripts/import/mapbox_country_pack.py @@ -1,6 +1,6 @@ import glob -from poly import parse_poly from shapely.geometry import Polygon +from shapely.io import from_geojson # directories used for searching for packages mapbox_meta_dir = 'mapbox/packages_meta' @@ -25,7 +25,7 @@ def getpackname(sname): # call with the name of POLY filename def country_pack(country_poly_fname): - country = parse_poly(country_poly_fname) + country = from_geojson(open(country_poly_fname, 'r').read()) packs = [] size_compressed = 0 size = 0 @@ -67,6 +67,6 @@ def world_pack(): if __name__ == '__main__': - print(country_pack('hierarchy/europe/poly')) + print(country_pack('hierarchy/north-america/us/pennsylvania/poly.json')) print() print(world_pack()) diff --git a/scripts/import/mapbox_planetiler_split.py b/scripts/import/mapbox_planetiler_split.py index 9cc51ff1..94d1ebe5 100755 --- a/scripts/import/mapbox_planetiler_split.py +++ b/scripts/import/mapbox_planetiler_split.py @@ -13,7 +13,8 @@ from collections import namedtuple, defaultdict from hierarchy import Hierarchy -from poly import parse_poly, intersects +from shapely.geometry import Polygon +from shapely.io import from_geojson # Note that files are formed according to XYZ notation while tiles @@ -67,7 +68,10 @@ def tri2mt(tri): def mt2bnd(mt): b = mercantile.bounds(mt) - return [b.west, b.south, b.east, b.north] + coors = [b.west, b.south, b.east, b.north] + poly = Polygon( ( (coors[0], coors[1]), (coors[0], coors[3]), + (coors[2], coors[3]), (coors[2], coors[1]) ) ) + return poly, [b.west, b.south, b.east, b.north] def splitter(ti): # determine into which file will the tile will go @@ -92,6 +96,12 @@ def mem_report(tag, mem_after, mem_before = 0): delta = (mem_after - mem_before)/1024/1024 print(f"Consumed memory {tag}: {delta:.1f}MB") +def intersects(polys, b): + for p in polys: + if p.intersects(b): + return True + return False + ## MAIN ## parser = argparse.ArgumentParser(description='Split MBTiles into smaller sub-packages.') @@ -112,10 +122,11 @@ def mem_report(tag, mem_after, mem_before = 0): for root, folders, files in os.walk(args.hierarchy): if "name" in files and not Hierarchy.ignore(root): name = Hierarchy.get_full_name(root) - poly = root + "/poly" + poly = root + "/poly.json" #print(Hierarchy.get_id(root), name) - polys.append(parse_poly(poly)) -print('Loaded hierarchy with {n} POLY files'.format(n=len(polys))) + cp = from_geojson(open(poly, 'r').read()) + polys.append(cp) +print('Loaded hierarchy with {n} POLY.JSON files'.format(n=len(polys))) # access to db print('Opening', args.mbtiles) @@ -146,7 +157,7 @@ def mem_report(tag, mem_after, mem_before = 0): filtered = dict() for tri,v in tofile.items(): k = tri2mt(tri) - b = mt2bnd(k) + b, _ = mt2bnd(k) if intersects(polys, b): f = fname(k) print('Keeping', k, f) @@ -161,7 +172,7 @@ def mem_report(tag, mem_after, mem_before = 0): counter = 0 for k, tiles in filtered.items(): f = fname(k) - bnd = mt2bnd(k) + _, bnd = mt2bnd(k) center = [(bnd[0]+bnd[2])/2, (bnd[1]+bnd[3])/2] ready = 100.0*counter/len(filtered) counter += 1 diff --git a/scripts/import/poly.py b/scripts/import/poly.py deleted file mode 100644 index dc207a5d..00000000 --- a/scripts/import/poly.py +++ /dev/null @@ -1,65 +0,0 @@ -import glob -from shapely.geometry import MultiPolygon, Polygon - -# taken from http://wiki.openstreetmap.org/wiki/Osmosis/Polygon_Filter_File_Python_Parsing -def parse_poly(fname): - """ Parse an Osmosis polygon filter file. - - Accept a sequence of lines from a polygon file, return a shapely.geometry.MultiPolygon object. - - http://wiki.openstreetmap.org/wiki/Osmosis/Polygon_Filter_File_Format - """ - in_ring = False - coords = [] - - lines = open(fname, 'r') - - for (index, line) in enumerate(lines): - if index == 0: - # first line is junk. - continue - - elif index == 1: - # second line is the first polygon ring. - coords.append([[], []]) - ring = coords[-1][0] - in_ring = True - - elif in_ring and line.strip() == 'END': - # we are at the end of a ring, perhaps with more to come. - in_ring = False - - elif in_ring: - # we are in a ring and picking up new coordinates. - ring.append(list(map(float, line.split()))) - - elif not in_ring and line.strip() == 'END': - # we are at the end of the whole polygon. - break - - elif not in_ring and line.startswith('!'): - # we are at the start of a polygon part hole. - coords[-1][1].append([]) - ring = coords[-1][1][-1] - in_ring = True - - elif not in_ring: - # we are at the start of a polygon part. - coords.append([[], []]) - ring = coords[-1][0] - in_ring = True - - return MultiPolygon(coords) - -def intersects(areas, coors): - poly = Polygon( ( (coors[0], coors[1]), (coors[0], coors[3]), - (coors[2], coors[3]), (coors[2], coors[1]) ) ) - for a in areas: - if a.intersects(poly): - return True - return False - -if __name__ == '__main__': - area = [parse_poly("hierarchy/europe/estonia/poly")] - i = intersects(area, [25, 57, 26, 60]) - print("intersects: ", i) diff --git a/scripts/import/prepare_countries.py b/scripts/import/prepare_countries.py index 0677e3d7..80d6b207 100755 --- a/scripts/import/prepare_countries.py +++ b/scripts/import/prepare_countries.py @@ -37,7 +37,7 @@ def pbfname(name): for root, folders, files in os.walk(Hierarchy.base_dir): if "name" in files and not Hierarchy.ignore(root): name = Hierarchy.get_full_name(root) - poly = root + "/poly" + polyjson = root + "/poly.json" print(Hierarchy.get_id(root), name, Hierarchy.get_postal(root), Hierarchy.get_postcodes(root)) cid = Hierarchy.get_id(root) @@ -48,12 +48,11 @@ def pbfname(name): #"osmscout": { "path": "osmscout/" + spath(cid) }, "geocoder_nlp": { "path": "geocoder-nlp/" + spath(cid) }, "mapnik_country": { "path": "mapnik/countries/" + spath(cid) }, - "mapboxgl_country": mapbox_pack(poly), - "valhalla": valhalla_pack(poly), + "mapboxgl_country": mapbox_pack(polyjson), + "valhalla": valhalla_pack(polyjson), } pbf = "$(PBF_DIR)/" + pbfname(cid) - polyjson = poly + ".json" # geocoder-nlp country_target = "$(BASE_DIR)/geocoder-nlp/" + spath(cid) + ".timestamp" diff --git a/scripts/import/prepare_distribution.py b/scripts/import/prepare_distribution.py index 27d48172..38252a2e 100755 --- a/scripts/import/prepare_distribution.py +++ b/scripts/import/prepare_distribution.py @@ -15,14 +15,14 @@ "base": url_base, "type": "url", #"osmscout": "osmscout-27", - "geocoder_nlp": "geocoder-nlp-36", + "geocoder_nlp": "geocoder-nlp-37", "postal_global": "postal-global-2", "postal_country": "postal-country-2", "mapnik_global": "mapnik-global-1", - "mapboxgl_country": "mapboxgl-22", - "mapboxgl_global": "mapboxgl-22", - "mapboxgl_glyphs": "mapboxgl-22", - "valhalla": "valhalla-30", + "mapboxgl_country": "mapboxgl-23", + "mapboxgl_global": "mapboxgl-23", + "mapboxgl_glyphs": "mapboxgl-23", + "valhalla": "valhalla-31", # mapnik is not updated anymore as users are expected to use vector tiles # keep this version just if someone needs raster tiles diff --git a/scripts/import/valhalla_country_pack.py b/scripts/import/valhalla_country_pack.py index 5fee1f86..f274563d 100644 --- a/scripts/import/valhalla_country_pack.py +++ b/scripts/import/valhalla_country_pack.py @@ -1,7 +1,6 @@ - import glob -from poly import parse_poly from shapely.geometry import Polygon +from shapely.io import from_geojson # directories used for searching for packages valhalla_meta_dir = 'valhalla/packages_meta' @@ -19,7 +18,7 @@ def gettimestamp(sname): # call with the name of POLY filename def country_pack(country_poly_fname): - country = parse_poly(country_poly_fname) + country = from_geojson(open(country_poly_fname, 'r').read()) packs = [] size_compressed = 0 size = 0 @@ -49,4 +48,4 @@ def country_pack(country_poly_fname): "size-compressed": str(size_compressed) } if __name__ == '__main__': - print(country_pack('hierarchy/europe/estonia/poly')) + print(country_pack('hierarchy/north-america/us/pennsylvania/poly.json'))