Fixes and new raw files parser

mapbuilder/builder.py

@@ -1,10 +1,12 @@
 import logging
+import os
 from datetime import UTC, datetime
 from pathlib import Path
 
 from .cache import Cache
 from .data.aixm2 import parse_aixm
 from .data.kml import KMLParser
+from .data.raw import RAWParser
 from .data.sidstar import parse_sidstar
 from .dfs import aixm
 from .handlers.jinja import JinjaHandler
@@ -38,10 +40,15 @@ def __init__(self, source_dir, target_dir, cache_dir, config):
                 self.data[data_source] = parser.parse()
             elif data_source_type == "raw":
                 logging.debug(f"Loading raw source {data_source}...")
-                with (source_dir / config["data"][data_source]["source"]).open(
-                    encoding="iso-8859-1",
-                ) as f:
-                    self.data[data_source] = f.read()
+                if os.path.isfile(source_dir / config["data"][data_source]["source"]):
+                    with (source_dir / config["data"][data_source]["source"]).open(
+                        encoding="iso-8859-1",
+                    ) as f:
+                        self.data[data_source] = f.read()
+                else:
+                    parser = RAWParser(source_dir / config["data"][data_source]["source"])
+                    self.data[data_source] = parser.parse()
+                    #logging.debug(self.data[data_source])
             elif data_source_type == "ese":
                 logging.debug(f"Loading ESE source {data_source}...")
                 self.data[data_source] = {
@@ -52,6 +59,7 @@ def __init__(self, source_dir, target_dir, cache_dir, config):
 
         self.jinja_handler = JinjaHandler(self.data, self.config)
 
+
     def __load_aixm(self, name: str, src: str):
         if src.startswith("aixm:dfs"):
             _, _, amdt, leaf = src.split(":")

mapbuilder/data/kml.py

@@ -1,6 +1,8 @@
+
 from pathlib import Path
 
 import xmltodict
+import logging
 from shapely import LinearRing, LineString, Point
 
 
@@ -27,8 +29,12 @@ def parse_pos_list(raw_geometry):
 
     def parse(self):
         result = {}
-        self.parse_recursively(self.xml_root["kml"]["Document"], result)
+        if "Document" in self.xml_root["kml"]:
+            self.parse_recursively(self.xml_root["kml"]["Document"], result)
+        else:
+            self.parse_recursively(self.xml_root["kml"], result)
         self.result = result
+        #logging.debug(result)
         if self.root is not None and self.root in self.result:
             self.result = self.result[self.root]
         return self.result
@@ -42,21 +48,69 @@ def parse_recursively(self, root, result):
 
         if "Placemark" in root:
             for placemark in ensure_list(root["Placemark"]):
-                if "LineString" in placemark:
+                if "MultiGeometry" in placemark:                
+                    name = placemark["name"]
+                    result[name] = {}
+                    self.parse_recursively(placemark["MultiGeometry"], result[name])
+                elif "LineString" in placemark:
                     geom = LineString(self.parse_pos_list(placemark["LineString"]["coordinates"]))
+                    name = placemark["name"]
+                    if name not in result:
+                        result[name] = [geom]
+                    else:
+                        result[name].append(geom)
                 elif "Polygon" in placemark:
                     geom = LinearRing(
                         self.parse_pos_list(
                             placemark["Polygon"]["outerBoundaryIs"]["LinearRing"]["coordinates"],
                         ),
                     )
+                    name = placemark["name"]
+                    if name not in result:
+                        result[name] = [geom]
+                    else:
+                        result[name].append(geom)
                 elif "Point" in placemark:
                     geom = Point(self.parse_pos_list(placemark["Point"]["coordinates"]))
+                    name = placemark["name"]
+                    if name not in result:
+                        result[name] = [geom]
+                    else:
+                        result[name].append(geom)
                 else:
                     msg = f"Placemark {placemark} unknown"
-                    raise ValueError(msg)
+                    raise ValueError(msg)             
+
+
+        if "LineString" in root:
+            for linestring in ensure_list(root["LineString"]):
+                geom = LineString(self.parse_pos_list(linestring["coordinates"]))
+
+                name = ""
+                if name not in result:
+                    result[name] = [geom]
+                else:
+                    result[name].append(geom)
+
+        if "Polygon" in root:
+            for polygon in ensure_list(root["Polygon"]):
+                geom = LinearRing(
+                        self.parse_pos_list(
+                           polygon["outerBoundaryIs"]["LinearRing"]["coordinates"],
+                        ),
+                    )
+
+                name = ""
+                if name not in result:
+                    result[name] = [geom]
+                else:
+                    result[name].append(geom) 
+
+        if "Point" in root:
+            for point in ensure_list(root["Point"]):
+                geom = Point(self.parse_pos_list(point["coordinates"]))
 
-                name = placemark["name"]
+                name = ""
                 if name not in result:
                     result[name] = [geom]
                 else:

mapbuilder/data/raw.py

@@ -0,0 +1,32 @@
+
+from pathlib import Path
+
+import logging
+import os
+
+class RAWParser:
+    result: dict | None
+
+    def __init__(self, file: Path):
+        self.result = None
+        self.file = file
+
+    def parse(self):
+        result = {}
+        self.parse_recursively(self.file, result)
+        self.result = result
+        #logging.debug(result)
+        return self.result
+
+    def parse_recursively(self, root, result):
+        for dirname in os.listdir(root):
+            #logging.debug(f"1: {root / dirname}")
+            if os.path.isfile(root / dirname):
+                filepath = root / dirname
+                with (root / dirname).open(
+                    encoding="iso-8859-1",
+                ) as f:
+                    result[Path(*filepath.parts[1:]).as_posix()] = f.read()
+            elif os.path.isdir(root / dirname):
+                #logging.debug(f"2: {dirname}")
+                self.parse_recursively(root / dirname, result)

mapbuilder/handlers/jinja.py

@@ -1,5 +1,7 @@
 from pathlib import Path
 
+import re
+import logging
 import numpy as np
 import shapely
 import shapely.ops
@@ -81,7 +83,6 @@ def concat(base: dict, keys: list[str]) -> list:
 
     return result
 
-
 def to_text_buffer(geometry, label: str, color: str, adapt_to_length=True):
     point = geometry[0] if isinstance(geometry, list) else geometry
 
@@ -97,6 +98,9 @@ def to_text_buffer(geometry, label: str, color: str, adapt_to_length=True):
 
     _render_polygon(lines, [buffer], color)
 
+    for i in range(len(lines)):
+        lines[i] = lines[i].replace("COLOR", "\nCOLOR")
+
     return "\n".join(lines)
 
 
@@ -107,7 +111,7 @@ def to_text(geometry, label: str):
         return ""
 
     labeltext, _, _ = label.partition("#")
-    return f"TEXT:{coord2es(point.coords[0])}:{labeltext}"
+    return f"TEXT:{coord2es(point.coords[0])}:{labeltext}\n"
 
 
 def to_symbol(geometry, symbol):
@@ -173,20 +177,20 @@ def envelope(geometries):
 def to_poly(geometries, designator: str, color: str, coordpoly=False):
     lines = [f"// {designator}"] if designator else []
 
-    _render_polygon(lines, _get_geoms(geometries), color)
-
-    if coordpoly:
-        lines.extend((f"COORDPOLY:{coordpoly}", ""))
+    _render_polygon(lines, _get_geoms(geometries), color, coordpoly)
 
     return "\n".join(lines)
 
 
-def _render_polygon(lines, polygons, color):
+def _render_polygon(lines, polygons, color, coordpoly=False):
     for polygon in polygons:
         lines.append(f"COLOR:{color}")
 
         for point in polygon.coords:
             lines.append(f"COORD:{coord2es((point[0], point[1]))}")
+
+        if coordpoly:
+            lines.extend((f"COORDPOLY:{coordpoly}", ""))
 
 
 def _render_coords(lines, linestring):