Add weather data infographic [don't merge]

anyway/db_views.py

@@ -56,6 +56,7 @@ class Views(object):
                                     markers.km,
                                     markers.km_raw,
                                     markers.km_accurate,
+                                    accident_weathers.rain_rate as accident_rain_rate,
                                     road_segments.segment_id as road_segment_id,
                                     road_segments.segment as road_segment_number,
                                     road_segments.from_name || ' - ' || road_segments.to_name as road_segment_name,
@@ -121,6 +122,7 @@ class Views(object):
                                      LEFT JOIN road_light ON markers.road_light = road_light.id AND markers.accident_year = road_light.year AND markers.provider_code = road_light.provider_code
                                      LEFT JOIN road_control ON markers.road_control = road_control.id AND markers.accident_year = road_control.year AND markers.provider_code = road_control.provider_code
                                      LEFT JOIN weather ON markers.weather = weather.id AND markers.accident_year = weather.year AND markers.provider_code = weather.provider_code
+                                     LEFT JOIN accident_weathers ON accident_weathers.provider_code = markers.provider_code AND accident_weathers.accident_id = markers.id AND accident_weathers.accident_year = markers.accident_year
                                      LEFT JOIN road_surface ON markers.road_surface = road_surface.id AND markers.accident_year = road_surface.year AND markers.provider_code = road_surface.provider_code
                                      LEFT JOIN road_object ON markers.road_object = road_object.id AND markers.accident_year = road_object.year AND markers.provider_code = road_object.provider_code
                                      LEFT JOIN object_distance ON markers.object_distance = object_distance.id AND markers.accident_year = object_distance.year AND markers.provider_code = object_distance.provider_code

anyway/infographics_utils.py

@@ -49,6 +49,7 @@ class WidgetId(Enum):
     vision_zero = auto()
     accident_count_by_driver_type = auto()
     accident_count_by_car_type = auto()
+    rain_accidents_by_severity = auto()
     injured_accidents_with_pedestrians = auto()
     accident_severity_by_cross_location = auto()
     motorcycle_accidents_vs_all_accidents = auto()
@@ -735,8 +736,10 @@ def __init__(self, request_params: RequestParams):
         }
 
     def generate_items(self) -> None:
-        self.items = AccidentCountByCarTypeWidget.get_stats_accidents_by_car_type_with_national_data(
-            self.request_params
+        self.items = (
+            AccidentCountByCarTypeWidget.get_stats_accidents_by_car_type_with_national_data(
+                self.request_params
+            )
         )
 
     @staticmethod
@@ -759,8 +762,10 @@ def get_stats_accidents_by_car_type_with_national_data(
         data_by_segment = AccidentCountByCarTypeWidget.percentage_accidents_by_car_type(
             involved_by_vehicle_type_data
         )
-        national_data = AccidentCountByCarTypeWidget.percentage_accidents_by_car_type_national_data_cache(
-            start_time, end_time
+        national_data = (
+            AccidentCountByCarTypeWidget.percentage_accidents_by_car_type_national_data_cache(
+                start_time, end_time
+            )
         )
 
         for k, v in national_data.items():  # pylint: disable=W0612
@@ -1041,6 +1046,66 @@ def pedestrian_injured_in_junctions_mock_data():  # Temporary for Frontend
         ]
 
 
+@WidgetCollection.register
+class AccidentCausedByRainWidget(Widget):
+    # the rain rate threshold after which we count the accident as a cause of the rain
+    ACCIDENT_RAIN_RATE_THRESHOLD = 4
+
+    def __init__(self, request_params: RequestParams):
+        super().__init__(request_params, WidgetId.rain_accidents_by_severity)
+        self.rank = 24
+        self.text = {
+            "title": "תאונות שהתרחשו בזמן גשם במקטע "
+            + self.request_params.location_info["road_segment_name"]
+        }
+
+    def generate_items(self) -> None:
+        self.items = AccidentCausedByRainWidget.stats_accidents_caused_by_rain_by_severity(
+            self.request_params.location_info,
+            self.request_params.start_time,
+            self.request_params.end_time,
+        )
+
+    @staticmethod
+    def stats_accidents_caused_by_rain_by_severity(location_info, start_time, end_time):
+        all_segment_accidents = get_accidents_stats(
+            table_obj=AccidentMarkerView,
+            filters=location_info,
+            start_time=start_time,
+            end_time=end_time,
+            raw=True,
+        )
+
+        severity_to_severity_hebrew = {}
+        accidents_by_severity = defaultdict(int)
+        rain_accidents_by_severity = defaultdict(int)
+        for accident in all_segment_accidents:
+            severity = accident["accident_severity"]
+            severity_hebrew = accident["accident_severity_hebrew"]
+            severity_to_severity_hebrew[severity] = severity_hebrew
+            accidents_by_severity[severity] += 1
+            if (
+                accident["accident_rain_rate"]
+                > AccidentCausedByRainWidget.ACCIDENT_RAIN_RATE_THRESHOLD
+            ):
+                rain_accidents_by_severity[severity] += 1
+
+        stats = []
+        for severity, rain_accidents_amount in rain_accidents_by_severity.items():
+            stats.append(
+                {
+                    "severity": severity,
+                    "severity_hebrew": severity_to_severity_hebrew[severity],
+                    "amount_of_accidents_caused_by_rain": rain_accidents_amount,
+                    "accidents_caused_by_rain_percentage": int(
+                        rain_accidents_amount / accidents_by_severity[severity] * 100
+                    ),
+                }
+            )
+
+        return stats
+
+
 def extract_news_flash_location(news_flash_obj):
     resolution = news_flash_obj.resolution or None
     if not news_flash_obj or not resolution or resolution not in resolution_dict:
@@ -1074,7 +1139,7 @@ def get_query(table_obj, filters, start_time, end_time):
 
 
 def get_accidents_stats(
-    table_obj, filters=None, group_by=None, count=None, start_time=None, end_time=None
+    table_obj, filters=None, group_by=None, count=None, start_time=None, end_time=None, raw=False
 ):
     filters = filters or {}
     filters["provider_code"] = [
@@ -1088,10 +1153,9 @@ def get_accidents_stats(
         query = query.with_entities(group_by, func.count(count))
     df = pd.read_sql_query(query.statement, query.session.bind)
     df.rename(columns={"count_1": "count"}, inplace=True)  # pylint: disable=no-member
-    df.columns = [c.replace("_hebrew", "") for c in df.columns]
-    return (  # pylint: disable=no-member
-        df.to_dict(orient="records") if group_by or count else df.to_dict()
-    )
+    if not raw:
+        df.columns = [c.replace("_hebrew", "") for c in df.columns]
+    return df.to_dict(orient="records")  # pylint: disable=no-member
 
 
 def get_injured_filters(location_info):

anyway/models.py

@@ -239,6 +239,7 @@ class AccidentMarker(MarkerMixin, Base):
     cross_direction = Column(Integer())
     involved = relationship("Involved")
     vehicles = relationship("Vehicle")
+    weather_data = relationship("AccidentWeather", uselist=False)
     video_link = Column(Text())
     road1 = Column(Integer())
     road2 = Column(Integer())
@@ -649,6 +650,47 @@ def parse(cls, data):
         )
 
 
+class AccidentWeather(Base):
+    __tablename__ = "accident_weathers"
+    id = Column(BigInteger(), primary_key=True)
+    provider_and_id = Column(BigInteger())
+    provider_code = Column(Integer())
+    accident_id = Column(BigInteger())
+    accident_year = Column(Integer())
+    rain_rate = Column(Integer())
+    __table_args__ = (
+        ForeignKeyConstraint(
+            [accident_id, provider_code, accident_year],
+            [AccidentMarker.id, AccidentMarker.provider_code, AccidentMarker.accident_year],
+            ondelete="CASCADE",
+        ),
+        Index("accident_id_idx_accident_weather", "accident_id", unique=False),
+        Index("provider_and_id_idx_accident_weather", "provider_and_id", unique=False),
+        {},
+    )
+
+    def serialize(self):
+        return {
+            "id": self.id,
+            "provider_code": self.provider_code,
+            "accident_id": self.accident_id,
+            "rain_rate": self.rain_rate,
+        }
+
+    # Flask-Login integration
+    def is_authenticated(self):
+        return True
+
+    def is_active(self):
+        return True
+
+    def is_anonymous(self):
+        return False
+
+    def get_id(self):
+        return self.id
+
+
 class DiscussionMarker(MarkerMixin, Base):
     __tablename__ = "discussions"
     __table_args__ = (
@@ -1772,6 +1814,7 @@ class AccidentMarkerView(Base):
     road_segment_id = Column(Integer())
     road_segment_name = Column(Text())
     road_segment_number = Column(Integer())
+    accident_rain_rate = Column(Integer())
 
     def serialize(self):
         return {
@@ -1866,6 +1909,7 @@ def serialize(self):
             "longitude": self.longitude,
             "x": self.x,
             "y": self.y,
+            "accident_rain_rate": self.accident_rain_rate,
         }
 
 

anyway/parsers/cbs/executor.py

@@ -15,6 +15,7 @@
 from sqlalchemy import or_, and_
 
 from anyway.parsers.cbs import preprocessing_cbs_files, importmail_cbs
+from anyway.parsers.cbs.weather_data import ensure_accidents_weather_data
 from anyway import field_names, localization
 from anyway.backend_constants import BE_CONST
 from anyway.models import (
@@ -1146,6 +1147,8 @@ def main(
         )
         logging.info("Total: {0} items in {1}".format(total, time_delta(started)))
 
+        ensure_accidents_weather_data()
+
         create_views()
     except Exception as ex:
         print("Exception occured while loading the cbs data: {0}".format(str(ex)))

anyway/parsers/cbs/weather_data.py

@@ -0,0 +1,48 @@
+import logging
+
+from anyway.app_and_db import db
+from anyway.models import (
+    AccidentMarker,
+    AccidentWeather,
+)
+from anyway.parsers.cbs.weather_interpolator import get_weather
+
+
+def ensure_accidents_weather_data(start_date=None, filters=None):
+    """
+    :param start_date: Add start date filter to the query that lists accident markers to add weather data to
+    :param filters: additional filters to add to the query that lists accident markers to add weather data to
+    This is used mainly for testing - format DD-MM-YYYY
+    :returns: int representing the number of accidents to which weather data was added
+    """
+    logging.info(f"Ensuring accidents weather data {start_date} {filters}")
+    query = db.session.query(AccidentMarker).filter(AccidentMarker.weather_data == None)
+    if start_date:
+        query = query.filter(AccidentMarker.created > start_date)
+    if filters is not None:
+        query = query.filter(*filters)
+    accident_markers_to_update = query.all()
+    if accident_markers_to_update:
+        logging.debug(
+            f"Found accident markers without weather data. {len(accident_markers_to_update)}"
+        )
+    accidents_weather_data = []
+    for accident_marker in query.all():
+        weather_data = get_weather(
+            accident_marker.latitude, accident_marker.longitude, accident_marker.created.isoformat()
+        )
+        accidents_weather_data.append(
+            {
+                "accident_id": accident_marker.id,
+                "provider_and_id": accident_marker.provider_and_id,
+                "provider_code": accident_marker.provider_code,
+                "accident_year": accident_marker.accident_year,
+                "rain_rate": weather_data["rain"],
+            }
+        )
+    if accidents_weather_data:
+        logging.debug(f"Adding weather data to accidents. {accidents_weather_data}")
+        db.session.bulk_insert_mappings(AccidentWeather, accidents_weather_data)
+        db.session.commit()
+    logging.debug("Finished filling accidents weather data")
+    return len(accident_markers_to_update) if accident_markers_to_update else 0