From 9feb649f9232b55bec3d62c992671163468e7c3e Mon Sep 17 00:00:00 2001 From: Sergei Dutenhefner <74149595+sergej-singer@users.noreply.github.com> Date: Mon, 3 Jun 2024 21:54:15 +0200 Subject: [PATCH] Fixes and new raw files parser KML Parser: - Added MultiGeometry Support - Added Check for existance of Tag jinja Handler: - Fixed: "COLOR:" appearing on the previous line instead of on the next line while rendering Buffer for text - Fixed: "TEXT:" appearing on the previous line instead of on the next line - Fixed: "COORDPOLY:" not appearing after every polygon while rendering Polygons for TopSky Maps New raw file parser: - Added the ability to load not only one raw file, but the whole directory --- mapbuilder/builder.py | 16 +++++++--- mapbuilder/data/kml.py | 62 +++++++++++++++++++++++++++++++++--- mapbuilder/data/raw.py | 32 +++++++++++++++++++ mapbuilder/handlers/jinja.py | 18 +++++++---- 4 files changed, 113 insertions(+), 15 deletions(-) create mode 100644 mapbuilder/data/raw.py diff --git a/mapbuilder/builder.py b/mapbuilder/builder.py index d553710..18f3e8f 100644 --- a/mapbuilder/builder.py +++ b/mapbuilder/builder.py @@ -1,10 +1,12 @@ import logging +import os from datetime import UTC, datetime from pathlib import Path from .cache import Cache from .data.aixm2 import parse_aixm from .data.kml import KMLParser +from .data.raw import RAWParser from .data.sidstar import parse_sidstar from .dfs import aixm from .handlers.jinja import JinjaHandler @@ -38,10 +40,15 @@ def __init__(self, source_dir, target_dir, cache_dir, config): self.data[data_source] = parser.parse() elif data_source_type == "raw": logging.debug(f"Loading raw source {data_source}...") - with (source_dir / config["data"][data_source]["source"]).open( - encoding="iso-8859-1", - ) as f: - self.data[data_source] = f.read() + if os.path.isfile(source_dir / config["data"][data_source]["source"]): + with (source_dir / config["data"][data_source]["source"]).open( + encoding="iso-8859-1", + ) as f: + self.data[data_source] = f.read() + else: + parser = RAWParser(source_dir / config["data"][data_source]["source"]) + self.data[data_source] = parser.parse() + #logging.debug(self.data[data_source]) elif data_source_type == "ese": logging.debug(f"Loading ESE source {data_source}...") self.data[data_source] = { @@ -52,6 +59,7 @@ def __init__(self, source_dir, target_dir, cache_dir, config): self.jinja_handler = JinjaHandler(self.data, self.config) + def __load_aixm(self, name: str, src: str): if src.startswith("aixm:dfs"): _, _, amdt, leaf = src.split(":") diff --git a/mapbuilder/data/kml.py b/mapbuilder/data/kml.py index e19bbb3..e6f211c 100644 --- a/mapbuilder/data/kml.py +++ b/mapbuilder/data/kml.py @@ -1,6 +1,8 @@ + from pathlib import Path import xmltodict +import logging from shapely import LinearRing, LineString, Point @@ -27,8 +29,12 @@ def parse_pos_list(raw_geometry): def parse(self): result = {} - self.parse_recursively(self.xml_root["kml"]["Document"], result) + if "Document" in self.xml_root["kml"]: + self.parse_recursively(self.xml_root["kml"]["Document"], result) + else: + self.parse_recursively(self.xml_root["kml"], result) self.result = result + #logging.debug(result) if self.root is not None and self.root in self.result: self.result = self.result[self.root] return self.result @@ -42,21 +48,69 @@ def parse_recursively(self, root, result): if "Placemark" in root: for placemark in ensure_list(root["Placemark"]): - if "LineString" in placemark: + if "MultiGeometry" in placemark: + name = placemark["name"] + result[name] = {} + self.parse_recursively(placemark["MultiGeometry"], result[name]) + elif "LineString" in placemark: geom = LineString(self.parse_pos_list(placemark["LineString"]["coordinates"])) + name = placemark["name"] + if name not in result: + result[name] = [geom] + else: + result[name].append(geom) elif "Polygon" in placemark: geom = LinearRing( self.parse_pos_list( placemark["Polygon"]["outerBoundaryIs"]["LinearRing"]["coordinates"], ), ) + name = placemark["name"] + if name not in result: + result[name] = [geom] + else: + result[name].append(geom) elif "Point" in placemark: geom = Point(self.parse_pos_list(placemark["Point"]["coordinates"])) + name = placemark["name"] + if name not in result: + result[name] = [geom] + else: + result[name].append(geom) else: msg = f"Placemark {placemark} unknown" - raise ValueError(msg) + raise ValueError(msg) + + + if "LineString" in root: + for linestring in ensure_list(root["LineString"]): + geom = LineString(self.parse_pos_list(linestring["coordinates"])) + + name = "" + if name not in result: + result[name] = [geom] + else: + result[name].append(geom) + + if "Polygon" in root: + for polygon in ensure_list(root["Polygon"]): + geom = LinearRing( + self.parse_pos_list( + polygon["outerBoundaryIs"]["LinearRing"]["coordinates"], + ), + ) + + name = "" + if name not in result: + result[name] = [geom] + else: + result[name].append(geom) + + if "Point" in root: + for point in ensure_list(root["Point"]): + geom = Point(self.parse_pos_list(point["coordinates"])) - name = placemark["name"] + name = "" if name not in result: result[name] = [geom] else: diff --git a/mapbuilder/data/raw.py b/mapbuilder/data/raw.py new file mode 100644 index 0000000..c598673 --- /dev/null +++ b/mapbuilder/data/raw.py @@ -0,0 +1,32 @@ + +from pathlib import Path + +import logging +import os + +class RAWParser: + result: dict | None + + def __init__(self, file: Path): + self.result = None + self.file = file + + def parse(self): + result = {} + self.parse_recursively(self.file, result) + self.result = result + #logging.debug(result) + return self.result + + def parse_recursively(self, root, result): + for dirname in os.listdir(root): + #logging.debug(f"1: {root / dirname}") + if os.path.isfile(root / dirname): + filepath = root / dirname + with (root / dirname).open( + encoding="iso-8859-1", + ) as f: + result[Path(*filepath.parts[1:]).as_posix()] = f.read() + elif os.path.isdir(root / dirname): + #logging.debug(f"2: {dirname}") + self.parse_recursively(root / dirname, result) \ No newline at end of file diff --git a/mapbuilder/handlers/jinja.py b/mapbuilder/handlers/jinja.py index ca687d7..b59fd83 100644 --- a/mapbuilder/handlers/jinja.py +++ b/mapbuilder/handlers/jinja.py @@ -1,5 +1,7 @@ from pathlib import Path +import re +import logging import numpy as np import shapely import shapely.ops @@ -81,7 +83,6 @@ def concat(base: dict, keys: list[str]) -> list: return result - def to_text_buffer(geometry, label: str, color: str, adapt_to_length=True): point = geometry[0] if isinstance(geometry, list) else geometry @@ -97,6 +98,9 @@ def to_text_buffer(geometry, label: str, color: str, adapt_to_length=True): _render_polygon(lines, [buffer], color) + for i in range(len(lines)): + lines[i] = lines[i].replace("COLOR", "\nCOLOR") + return "\n".join(lines) @@ -107,7 +111,7 @@ def to_text(geometry, label: str): return "" labeltext, _, _ = label.partition("#") - return f"TEXT:{coord2es(point.coords[0])}:{labeltext}" + return f"TEXT:{coord2es(point.coords[0])}:{labeltext}\n" def to_symbol(geometry, symbol): @@ -173,20 +177,20 @@ def envelope(geometries): def to_poly(geometries, designator: str, color: str, coordpoly=False): lines = [f"// {designator}"] if designator else [] - _render_polygon(lines, _get_geoms(geometries), color) - - if coordpoly: - lines.extend((f"COORDPOLY:{coordpoly}", "")) + _render_polygon(lines, _get_geoms(geometries), color, coordpoly) return "\n".join(lines) -def _render_polygon(lines, polygons, color): +def _render_polygon(lines, polygons, color, coordpoly=False): for polygon in polygons: lines.append(f"COLOR:{color}") for point in polygon.coords: lines.append(f"COORD:{coord2es((point[0], point[1]))}") + + if coordpoly: + lines.extend((f"COORDPOLY:{coordpoly}", "")) def _render_coords(lines, linestring):