Magical java typing for additional attributes (#124)

* add assuming java types for additional attributes when writing to xml

* add assuming python types for additional java-typed attributes when reading from xml

* fix lingering dependence on long form attributes

* update notebooks with simple form attributes

* extra fixes, add dtype param to road pricing to match with OSM ID dtypes

* increase code coverage

* address PR comments

* PR update: enable forcing long form when reading matsim networks

* lint

example_data/pt2matsim_network/road_pricing/osm_to_network_ids.json

@@ -1 +1 @@
-{"26997928": ["1", "2", "3", "4"], "546461337": ["998", "999"]}
+{"26997928.0": ["1", "2", "3", "4"], "546461337.0": ["998", "999"]}

example_data/pt2matsim_network/road_pricing/osm_toll_id_ref.csv

@@ -1,3 +1,3 @@
 osm_id,osm_ref,osm_name
-26997928,A400,Charing Cross Road
-546461337,A3211,Byward Street
+26997928.0,A400,Charing Cross Road
+546461337.0,A3211,Byward Street

example_data/pt2matsim_network/road_pricing/osm_tolls_with_network_ids.csv

@@ -1,3 +1,3 @@
 ,osm_id,osm_ref,osm_name,network_id,vehicle_type,toll_amount,start_time,end_time
-0,26997928,A400,Charing Cross Road,True,type2,2.9,00:00,23:59
-1,546461337,A3211,Byward Street,True,type2,2.9,00:00,23:59
+0,26997928.0,A400,Charing Cross Road,True,type2,2.9,00:00,23:59
+1,546461337.0,A3211,Byward Street,True,type2,2.9,00:00,23:59

example_data/pt2matsim_network/road_pricing/roadpricing-file.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" ?>
 <!DOCTYPE roadpricing SYSTEM "http://www.matsim.org/files/dtd/roadpricing_v1.dtd">
-<roadpricing type="cordon" name="cordon-toll">
-  <description>A simple cordon toll scheme</description>
+<roadpricing type="link" name="simple-toll">
+  <description>A simple toll scheme</description>
   <links>
     <!-- === A3211 === -->
     <link id="998">

genet/core.py

@@ -41,7 +41,7 @@ def __init__(self, epsg, **kwargs):
         self.epsg = epsg
         self.transformer = Transformer.from_crs(epsg, 'epsg:4326', always_xy=True)
         self.graph = nx.MultiDiGraph(name='Network graph', crs=epsg)
-        self.attributes = {'crs': {'name': 'crs', 'class': 'java.lang.String', 'text': epsg}}
+        self.attributes = {'crs': epsg}
         self.schedule = schedule_elements.Schedule(epsg)
         self.change_log = change_log.ChangeLog()
         self.auxiliary_files = {'node': {}, 'link': {}}
@@ -262,11 +262,12 @@ def simplify(self, no_processes=1, keep_loops=False):
         self._mark_as_simplified()
 
     def _mark_as_simplified(self):
-        self.attributes['simplified'] = {'name': 'simplified', 'class': 'java.lang.String', 'text': 'true'}
+        self.attributes['simplified'] = True
 
     def is_simplified(self):
         if 'simplified' in self.attributes:
-            return self.attributes['simplified']['text'] in {'true', 'True', True}
+            # range of values for backwards compatibility
+            return self.attributes['simplified'] in {'true', 'True', True}
         return False
 
     def node_attribute_summary(self, data=False):

genet/exceptions.py

@@ -95,3 +95,10 @@ class PartialMaxStableSetProblem(Exception):
     Raised when the maximum stable set to snap PT to the network is partial - some stops found nothing to snap to
     """
     pass
+
+
+class MalformedAdditionalAttributeError(Exception):
+    """
+    Raised when additional attributes can not be saved to MATSim network
+    """
+    pass

genet/input/matsim_reader.py

@@ -8,6 +8,7 @@
 from genet.schedule_elements import Route, Stop, Service
 from genet.utils import dict_support
 from genet.utils import spatial
+from genet.utils import java_dtypes
 
 
 def read_node(elem, g, node_id_mapping, node_attribs, transformer):
@@ -89,8 +90,8 @@ def read_link(elem, g, u, v, node_id_mapping, link_id_mapping, link_attribs):
 
     if link_attribs:
         if 'geometry' in link_attribs:
-            if link_attribs['geometry']['text']:
-                attribs['geometry'] = spatial.decode_polyline_to_shapely_linestring(link_attribs['geometry']['text'])
+            if link_attribs['geometry']:
+                attribs['geometry'] = spatial.decode_polyline_to_shapely_linestring(link_attribs['geometry'])
                 del link_attribs['geometry']
         if link_attribs:
             attribs['attributes'] = link_attribs
@@ -103,27 +104,66 @@ def read_link(elem, g, u, v, node_id_mapping, link_id_mapping, link_attribs):
     return g, u, v, link_id_mapping, duplicated_link_id
 
 
-def update_additional_attrib(elem, attribs):
+def update_additional_attrib(elem, attribs, force_long_form_attributes=False):
     """
     Reads additional attributes
     :param elem:
     :param attribs: current additional attributes
+    :param force_long_form_attributes: Defaults to False, if True the additional attributes will be read into long form
     :return:
     """
-    attribs[elem.attrib['name']] = read_additional_attrib(elem)
+    attribs[elem.attrib['name']] = read_additional_attrib(elem, force_long_form_attributes=force_long_form_attributes)
     return attribs
 
 
-def read_additional_attrib(elem):
-    d = elem.attrib
+def read_additional_attrib(elem, force_long_form_attributes=False):
+    """
+    :param elem:
+    :param force_long_form_attributes: Defaults to False, if True the additional attributes will be read into long form
+    :return:
+    """
+    if force_long_form_attributes:
+        return _read_additional_attrib_to_long_form(elem)
+    else:
+        return _read_additional_attrib_to_short_form(elem)
+
+
+def _read_additional_attrib_text(elem):
     if elem.text is None:
-        d['text'] = ''
-        logging.warning(f"Elem {elem.attrib['name']} is being read as None.")
+        t = ''
+        logging.warning(f"Elem {elem.attrib['name']} is being read as None. Defaulting value to empty string.")
     elif (',' in elem.text) and elem.attrib['name'] != 'geometry':
-        d['text'] = set(elem.text.split(','))
+        t = set(elem.text.split(','))
+    else:
+        t = elem.text
+    return t
+
+
+def _read_additional_attrib_class(elem):
+    if 'class' in elem.attrib:
+        c = elem.attrib['class']
+    else:
+        logging.warning(f"Elem {elem.attrib['name']} does not have a Java class declared. "
+                        "Defaulting type to string.")
+        c = 'java.lang.String'
+    return c
+
+
+def _read_additional_attrib_to_short_form(elem):
+    t = _read_additional_attrib_text(elem)
+    if t and isinstance(t, str):
+        c = _read_additional_attrib_class(elem)
+        return java_dtypes.java_to_python_dtype(c)(t)
     else:
-        d['text'] = elem.text
-    return d
+        return t
+
+
+def _read_additional_attrib_to_long_form(elem):
+    return {
+        'text': _read_additional_attrib_text(elem),
+        'class': _read_additional_attrib_class(elem),
+        'name': elem.attrib['name']
+    }
 
 
 def unique_link_id(link_id, link_id_mapping):
@@ -141,11 +181,22 @@ def unique_link_id(link_id, link_id_mapping):
     return link_id, duplicated_link_id
 
 
-def read_network(network_path, transformer: Transformer):
+def read_network(network_path, transformer: Transformer, force_long_form_attributes=False):
     """
     Read MATSim network
     :param network_path: path to the network.xml file
     :param transformer: pyproj crs transformer
+    :param force_long_form_attributes: Defaults to False, if True the additional attributes will be read into verbose
+        format:
+            {
+                'additional_attrib': {'name': 'additional_attrib', 'class': 'java.lang.String', 'text': 'attrib_value'}
+            }
+        where 'attrib_value' is always a python string; instead of the default short form:
+            {
+                'additional_attrib': 'attrib_value'
+            }
+        where the type of attrib_value is mapped to a python type using the declared java class.
+        NOTE! Network level attributes cannot be forced to be read into long form.
     :return: g (nx.MultiDiGraph representing the multimodal network),
         node_id_mapping (dict {matsim network node ids : s2 spatial ids}),
         link_id_mapping (dict {matsim network link ids : {'from': matsim id from node, ,'to': matsim id to
@@ -193,19 +244,33 @@ def read_network(network_path, transformer: Transformer):
                 link_attribs = {}
             elif elem.tag == 'attribute':
                 if elem_type_for_additional_attributes == 'links':
-                    link_attribs = update_additional_attrib(elem, link_attribs)
+                    link_attribs = update_additional_attrib(elem, link_attribs, force_long_form_attributes)
                 elif elem_type_for_additional_attributes == 'network':
-                    network_attributes = update_additional_attrib(elem, network_attributes)
+                    if force_long_form_attributes:
+                        logging.warning('Network-level additional attributes are always read into short form.')
+                    network_attributes = update_additional_attrib(elem, network_attributes,
+                                                                  force_long_form_attributes=False)
                 elif elem_type_for_additional_attributes == 'nodes':
-                    node_attribs = update_additional_attrib(elem, node_attribs)
+                    node_attribs = update_additional_attrib(elem, node_attribs, force_long_form_attributes)
     return g, link_id_mapping, duplicated_node_ids, duplicated_link_ids, network_attributes
 
 
-def read_schedule(schedule_path, epsg):
+def read_schedule(schedule_path, epsg, force_long_form_attributes=False):
     """
     Read MATSim schedule
     :param schedule_path: path to the schedule.xml file
     :param epsg: 'epsg:12345'
+    :param force_long_form_attributes: Defaults to False, if True the additional attributes will be read into verbose
+        format:
+            {
+                'additional_attrib': {'name': 'additional_attrib', 'class': 'java.lang.String', 'text': 'attrib_value'}
+            }
+        where 'attrib_value' is always a python string; instead of the default short form:
+            {
+                'additional_attrib': 'attrib_value'
+            }
+        where the type of attrib_value is mapped to a python type using the declared java class.
+        NOTE! Schedule level attributes cannot be forced to be read into long form.
     :return: list of Service objects
     """
     services = []
@@ -349,24 +414,33 @@ def write_transitLinesTransitRoute(transitLine, transitRoutes, transportMode):
         elif event == 'end':
             if elem.tag == 'attribute':
                 if elem_type_for_additional_attributes == 'transitSchedule':
+                    if force_long_form_attributes:
+                        logging.warning('Schedule-level additional attributes are always read into short form.')
                     schedule_attribs = update_additional_attrib(elem, schedule_attribs)
                 elif elem_type_for_additional_attributes == 'stopFacility':
                     current_stop_data = transit_stop_id_mapping[current_stop_id]
                     if 'attributes' in current_stop_data:
                         current_stop_data['attributes'] = update_additional_attrib(
                             elem,
-                            transit_stop_id_mapping[current_stop_id]['attributes'])
+                            transit_stop_id_mapping[current_stop_id]['attributes'],
+                            force_long_form_attributes=force_long_form_attributes
+                        )
                     else:
-                        current_stop_data['attributes'] = update_additional_attrib(elem, {})
+                        current_stop_data['attributes'] = update_additional_attrib(
+                            elem,
+                            {},
+                            force_long_form_attributes=force_long_form_attributes)
                 elif elem_type_for_additional_attributes == 'transitLine':
                     transitLine['attributes'] = update_additional_attrib(
                         elem,
-                        transitLine['attributes']
+                        transitLine['attributes'],
+                        force_long_form_attributes=force_long_form_attributes
                     )
                 elif elem_type_for_additional_attributes == 'transitRoute':
                     transitRoutes[current_route_id]['attributes'] = update_additional_attrib(
                         elem,
-                        transitRoutes[current_route_id]['attributes']
+                        transitRoutes[current_route_id]['attributes'],
+                        force_long_form_attributes=force_long_form_attributes
                     )
             elif elem.tag == "transportMode":
                 transportMode = {'transportMode': elem.text}

genet/input/osm_reader.py

@@ -163,11 +163,7 @@ def generate_graph_edges(edges, reindexing_dict, nodes_and_attributes, config_pa
         link_attributes['attributes'] = {}
         for key, val in attribs.items():
             if key not in link_attributes:
-                link_attributes['attributes']['osm:way:{}'.format(key)] = {
-                    'name': 'osm:way:{}'.format(key),
-                    'class': 'java.lang.String',
-                    'text': str(val),
-                }
+                link_attributes['attributes'][f'osm:way:{key}'] = val
         edges_attributes.append(link_attributes)
     return edges_attributes
 

genet/input/read.py

@@ -16,33 +16,59 @@
 from genet.exceptions import NetworkSchemaError
 
 
-def read_matsim(path_to_network: str, epsg: str, path_to_schedule: str = None, path_to_vehicles: str = None):
+def read_matsim(path_to_network: str, epsg: str, path_to_schedule: str = None, path_to_vehicles: str = None,
+                force_long_form_attributes=False):
     """
     Reads MATSim's network.xml to genet.Network object and if give, also the schedule.xml and vehicles.xml into
     genet.Schedule object, part of the genet.Network object.
     :param path_to_network: path to MATSim's network.xml file
     :param path_to_schedule: path to MATSim's schedule.xml file, optional
     :param path_to_vehicles: path to MATSim's vehicles.xml file, optional, expected to be passed with a schedule
     :param epsg: projection for the network, e.g. 'epsg:27700'
+    :param force_long_form_attributes: Defaults to False, if True the additional attributes will be read into verbose
+        format:
+            {
+                'additional_attrib': {'name': 'additional_attrib', 'class': 'java.lang.String', 'text': 'attrib_value'}
+            }
+        where 'attrib_value' is always a python string; instead of the default short form:
+            {
+                'additional_attrib': 'attrib_value'
+            }
+        where the type of attrib_value is mapped to a python type using the declared java class.
+        NOTE! Network and Schedule level attributes cannot be forced to be read into long form.
     :return: genet.Network object
     """
-    n = read_matsim_network(path_to_network=path_to_network, epsg=epsg)
+    n = read_matsim_network(path_to_network=path_to_network, epsg=epsg,
+                            force_long_form_attributes=force_long_form_attributes)
     if path_to_schedule:
         n.schedule = read_matsim_schedule(
-            path_to_schedule=path_to_schedule, path_to_vehicles=path_to_vehicles, epsg=epsg)
+            path_to_schedule=path_to_schedule, path_to_vehicles=path_to_vehicles, epsg=epsg,
+            force_long_form_attributes=force_long_form_attributes)
     return n
 
 
-def read_matsim_network(path_to_network: str, epsg: str):
+def read_matsim_network(path_to_network: str, epsg: str, force_long_form_attributes=False):
     """
     Reads MATSim's network.xml to genet.Network object
     :param path_to_network: path to MATSim's network.xml file
     :param epsg: projection for the network, e.g. 'epsg:27700'
+    :param force_long_form_attributes: Defaults to False, if True the additional attributes will be read into verbose
+        format:
+            {
+                'additional_attrib': {'name': 'additional_attrib', 'class': 'java.lang.String', 'text': 'attrib_value'}
+            }
+        where 'attrib_value' is always a python string; instead of the default short form:
+            {
+                'additional_attrib': 'attrib_value'
+            }
+        where the type of attrib_value is mapped to a python type using the declared java class.
+        NOTE! Network level attributes cannot be forced to be read into long form.
     :return: genet.Network object
     """
     n = core.Network(epsg=epsg)
     n.graph, n.link_id_mapping, duplicated_nodes, duplicated_links, network_attributes = \
-        matsim_reader.read_network(path_to_network, n.transformer)
+        matsim_reader.read_network(path_to_network, n.transformer,
+                                   force_long_form_attributes=force_long_form_attributes)
     n.attributes = dict_support.merge_complex_dictionaries(n.attributes, network_attributes)
     n.graph.graph['crs'] = n.epsg
 
@@ -65,16 +91,28 @@ def read_matsim_network(path_to_network: str, epsg: str):
     return n
 
 
-def read_matsim_schedule(path_to_schedule: str, epsg: str, path_to_vehicles: str = None):
+def read_matsim_schedule(path_to_schedule: str, epsg: str, path_to_vehicles: str = None,
+                         force_long_form_attributes=False):
     """
     Reads MATSim's schedule.xml (and possibly vehicles.xml) to genet.Schedule object
     :param path_to_schedule: path to MATSim's schedule.xml file,
     :param path_to_vehicles: path to MATSim's vehicles.xml file, optional but encouraged
     :param epsg: projection for the schedule, e.g. 'epsg:27700'
+    :param force_long_form_attributes: Defaults to False, if True the additional attributes will be read into verbose
+        format:
+            {
+                'additional_attrib': {'name': 'additional_attrib', 'class': 'java.lang.String', 'text': 'attrib_value'}
+            }
+        where 'attrib_value' is always a python string; instead of the default short form:
+            {
+                'additional_attrib': 'attrib_value'
+            }
+        where the type of attrib_value is mapped to a python type using the declared java class.
+        NOTE! Schedule level attributes cannot be forced to be read into long form.
     :return: genet.Schedule object
     """
     services, minimal_transfer_times, transit_stop_id_mapping, schedule_attributes = matsim_reader.read_schedule(
-        path_to_schedule, epsg)
+        path_to_schedule, epsg, force_long_form_attributes=force_long_form_attributes)
     if path_to_vehicles:
         vehicles, vehicle_types = matsim_reader.read_vehicles(path_to_vehicles)
         matsim_schedule = schedule_elements.Schedule(