Merge pull request #2580 from ziv17/2567-use-segment-id-in-news-flash

2567 use segment id in news flash

anyway/flask_app.py

@@ -1159,6 +1159,7 @@ def datetime_to_str(val: datetime.datetime) -> str:
         "street2_hebrew": fields.String(),
         "non_urban_intersection_hebrew": fields.String(),
         "road_segment_name": fields.String(),
+        "road_segment_id": fields.Integer(),
         "newsflash_location_qualification": fields.Integer(),
         "location_qualifying_user": fields.Integer(),
     },

anyway/infographics_utils.py

@@ -20,7 +20,7 @@
 )
 from anyway.backend_constants import BE_CONST, AccidentType
 from anyway.models import NewsFlash, AccidentMarkerView
-from anyway.parsers import resolution_dict
+from anyway.parsers.resolution_fields import ResolutionFields as RF
 from anyway.infographics_dictionaries import head_on_collisions_comparison_dict
 from anyway.parsers import infographics_data_cache_updater
 from anyway.widgets.widget import Widget, widgets_dict
@@ -243,7 +243,7 @@ def is_news_flash_resolution_supported(news_flash_obj: NewsFlash) -> bool:
         return False
     location = location_data[DATA]
     for cat in BE_CONST.SUPPORTED_RESOLUTIONS:
-        if cat.value in resolution_dict and set(resolution_dict[cat.value]) <= location.keys():
+        if set(RF.get_required_fields(cat.value)) <= location.keys():
             return True
     return False
 

anyway/models.py

@@ -897,16 +897,16 @@ def set_critical(
         )
         from anyway.request_params import get_latest_accident_date, LocationInfo
 
-        if (self.road1 is None or self.road_segment_name is None) and (self.yishuv_name is None or self.street1_hebrew is None) :
+        if (self.road1 is None or self.road_segment_id is None) and (self.yishuv_name is None or self.street1_hebrew is None) :
             return None
         last_accident_date = get_latest_accident_date(table_obj=AccidentMarkerView, filters=None)
         resolution = BE_CONST.ResolutionCategories(self.resolution)
         end_time = last_accident_date.to_pydatetime().date()
         start_time = datetime.date(end_time.year + 1 - years_before, 1, 1)
         location_info = LocationInfo()
         if resolution == BE_CONST.ResolutionCategories.SUBURBAN_ROAD:
-            location_info["road1"]
-            location_info["road_segment_name"]
+            location_info["road1"] = self.road1
+            location_info["road_segment_id"] = self.road_segment_id
         elif resolution == BE_CONST.ResolutionCategories.STREET:
             location_info["yishuv_name"] = self.yishuv_name
             location_info["street1_hebrew"] = self.street1_hebrew
@@ -985,7 +985,7 @@ def serialize(self):
             "street1_hebrew": self.street1_hebrew,
             "street2_hebrew": self.street2_hebrew,
             "non_urban_intersection_hebrew": self.non_urban_intersection_hebrew,
-            "road_segment_name": self.road_segment_name,
+            "road_segment_id": self.road_segment_id,
             "newsflash_location_qualification": NewsflashLocationQualification(
                 self.newsflash_location_qualification
             ).get_label(),

anyway/parsers/__init__.py

@@ -1,26 +1,4 @@
-resolution_dict = {
-    "מחוז": ["region_hebrew"],
-    "נפה": ["district_hebrew"],
-    "עיר": ["yishuv_name"],
-    "רחוב": ["yishuv_name", "street1_hebrew"],
-    "צומת עירוני": ["yishuv_name", "street1_hebrew", "street2_hebrew"],
-    "כביש בינעירוני": ["road1", "road_segment_name"],
-    "צומת בינעירוני": ["non_urban_intersection", "non_urban_intersection_hebrew", "road1", "road2"],
-    "אחר": [
-        "region_hebrew",
-        "district_hebrew",
-        "yishuv_name",
-        "street1_hebrew",
-        "street2_hebrew",
-        "non_urban_intersection_hebrew",
-        "road1",
-        "road2",
-        "road_segment_name",
-        "road_segment_id",
-    ],
-}
-
-fields_to_resolution = {
-    ("yishuv_name", "street1_hebrew"): "רחוב",
-    ("road_segment_name", "road1"): "כביש בינעירוני",
-}
+"""
+resolution_dict is depracated. Its functionality was moved to class
+ResolutionFields, in module resolution_fields in same directory.
+"""

anyway/parsers/infographics_data_cache_updater.py

@@ -1,12 +1,8 @@
 # -*- coding: utf-8 -*-
 
 from datetime import datetime
-from sqlalchemy import not_
 from anyway.models import (
     Base,
-    InfographicsDataCache,
-    InfographicsDataCacheTemp,
-    NewsFlash,
     RoadSegments,
     InfographicsRoadSegmentsDataCache,
     InfographicsRoadSegmentsDataCacheTemp,
@@ -34,80 +30,13 @@
 META = "meta"
 DATA = "data"
 ITEMS = "items"
-REGULAR_CACHE_TABLES = {CACHE: InfographicsDataCache, TEMP: InfographicsDataCacheTemp}
 STREET_CACHE_TABLES = {CACHE: InfographicsStreetDataCache, TEMP: InfographicsStreetDataCacheTemp}
 ROAD_SEGMENT_CACHE_TABLES = {
     CACHE: InfographicsRoadSegmentsDataCache,
     TEMP: InfographicsRoadSegmentsDataCacheTemp,
 }
 
 
-def is_in_cache(nf):
-    return (
-        len(CONST.INFOGRAPHICS_CACHE_YEARS_AGO)
-        == db.session.query(InfographicsDataCache)
-        .filter(InfographicsDataCache.news_flash_id == nf.get_id())
-        .count()
-    )
-
-
-# noinspection PyUnresolvedReferences
-def add_news_flash_to_cache(news_flash: NewsFlash):
-    try:
-        if not (
-            news_flash.accident
-            and anyway.infographics_utils.is_news_flash_resolution_supported(news_flash)
-            and news_flash.newsflash_location_qualification is not None
-        ):
-            logging.debug(
-                f"add_news_flash_to_cache: news flash does not qualify:{news_flash.serialize()}"
-            )
-            return True
-        db.get_engine().execute(
-            InfographicsDataCache.__table__.insert(),  # pylint: disable=no-member
-            [
-                {
-                    "news_flash_id": news_flash.get_id(),
-                    "years_ago": y,
-                    "data": anyway.infographics_utils.create_infographics_data(
-                        news_flash.get_id(), y, "he"
-                    ),
-                }
-                for y in CONST.INFOGRAPHICS_CACHE_YEARS_AGO
-            ],
-        )
-        logging.info(f"{news_flash.get_id()} added to cache")
-        return True
-    except Exception as e:
-        logging.exception(
-            f"Exception while inserting to cache. flash_id:{news_flash}), cause:{e.__cause__}"
-        )
-        return False
-
-
-def get_infographics_data_from_cache(news_flash_id, years_ago) -> Dict:
-    db_item = (
-        db.session.query(InfographicsDataCache)
-        .filter(InfographicsDataCache.news_flash_id == news_flash_id)
-        .filter(InfographicsDataCache.years_ago == years_ago)
-        .first()
-    )
-    logging.debug(f"retrieved from cache {type(db_item)}:{db_item}"[:70])
-    db.session.commit()
-    try:
-        if db_item:
-            return json.loads(db_item.get_data())
-        else:
-            return {}
-    except Exception as e:
-        logging.error(
-            f"Exception while extracting data from returned cache item flash_id:{news_flash_id},years:{years_ago})"
-            f"returned value {type(db_item)}"
-            f":cause:{e.__cause__}, class:{e.__class__}"
-        )
-        return {}
-
-
 def get_infographics_data_from_cache_by_road_segment(road_segment_id, years_ago) -> Dict:
     db_item = (
         db.session.query(InfographicsRoadSegmentsDataCache)
@@ -236,33 +165,6 @@ def copy_temp_into_cache(table: Dict[str, Base]):
     db.session.commit()
 
 
-# noinspection PyUnresolvedReferences
-def build_cache_into_temp():
-    start = datetime.now()
-    db.session.query(InfographicsDataCacheTemp).delete()
-    db.session.commit()
-    supported_resolutions = set([x.value for x in BE_CONST.SUPPORTED_RESOLUTIONS])
-    for y in CONST.INFOGRAPHICS_CACHE_YEARS_AGO:
-        logging.debug(f"processing years_ago:{y}")
-        db.get_engine().execute(
-            InfographicsDataCacheTemp.__table__.insert(),  # pylint: disable=no-member
-            [
-                {
-                    "news_flash_id": new_flash.get_id(),
-                    "years_ago": y,
-                    "data": anyway.infographics_utils.create_infographics_data(
-                        new_flash.get_id(), y, "he"
-                    ),
-                }
-                for new_flash in db.session.query(NewsFlash)
-                .filter(NewsFlash.accident)
-                .filter(NewsFlash.resolution.in_(supported_resolutions))
-                .all()
-            ],
-        )
-    logging.info(f"cache rebuild took:{str(datetime.now() - start)}")
-
-
 def get_streets() -> Iterable[Streets]:
     t = Streets
     street_iter = iter(db.session.query(t.yishuv_symbol, t.street).all())
@@ -340,21 +242,6 @@ def build_road_segments_cache_into_temp():
     logging.info(f"cache rebuild took:{str(datetime.now() - start)}")
 
 
-def get_cache_info():
-    cache_items = db.session.query(InfographicsDataCache).count()
-    tmp_items = db.session.query(InfographicsDataCacheTemp).count()
-    num_acc_flash_items = db.session.query(NewsFlash).filter(NewsFlash.accident).count()
-    num_acc_suburban_flash_items = (
-        db.session.query(NewsFlash)
-        .filter(NewsFlash.accident)
-        .filter(NewsFlash.resolution.in_(["כביש בינעירוני"]))
-        .filter(not_(NewsFlash.road_segment_name == None))
-        .count()
-    )
-    db.session.commit()
-    return f"num items in cache: {cache_items}, temp table: {tmp_items}, accident flashes:{num_acc_flash_items}, flashes processed:{num_acc_suburban_flash_items}"
-
-
 def main_for_road_segments():
     logging.info("Refreshing road segments infographics cache...")
     build_road_segments_cache_into_temp()
@@ -365,15 +252,3 @@ def main_for_road_segments():
 def main_for_street():
     build_street_cache_into_temp()
     copy_temp_into_cache(STREET_CACHE_TABLES)
-
-
-def main(update, info):
-    if update:
-        logging.info("Refreshing infographics cache...")
-        build_cache_into_temp()
-        copy_temp_into_cache(REGULAR_CACHE_TABLES)
-        logging.info("Refreshing infographics cache Done")
-    if info:
-        logging.info(get_cache_info())
-    else:
-        logging.debug(f"{info}")

anyway/parsers/location_extraction.py

@@ -7,7 +7,7 @@
 from geographiclib.geodesic import Geodesic
 from anyway.backend_constants import BE_CONST
 from anyway.models import NewsFlash
-from anyway.parsers import resolution_dict
+from anyway.parsers.resolution_fields import ResolutionFields as RF
 from anyway import secrets
 from anyway.models import AccidentMarkerView, RoadSegments
 from sqlalchemy import not_
@@ -191,7 +191,7 @@ def get_db_matching_location(db, latitude, longitude, resolution, road_no=None):
     """
     # READ MARKERS FROM DB
     geod = Geodesic.WGS84
-    relevant_fields = resolution_dict[resolution]
+    relevant_fields = RF.get_possible_fields(resolution)
     markers = db.get_markers_for_location_extraction()
     markers["geohash"] = markers.apply(
         lambda x: geohash.encode(x["latitude"], x["longitude"], precision=4), axis=1
@@ -538,12 +538,9 @@ def extract_geo_features(db, newsflash: NewsFlash, update_cbs_location_only: boo
     if location_from_db is not None:
         for k, v in location_from_db.items():
             setattr(newsflash, k, v)
-        all_resolutions = []
-        for _, v in resolution_dict.items():
-            all_resolutions += v
-        for resolution in all_resolutions:
-            if resolution not in location_from_db:
-                setattr(newsflash, resolution, None)
+        for field in RF.get_all_location_fields():
+            if field not in location_from_db:
+                setattr(newsflash, field, None)
     if (
         newsflash.road_segment_id is None
         and newsflash.road_segment_name is not None

anyway/parsers/news_flash_db_adapter.py

@@ -5,7 +5,6 @@
 import numpy as np
 from sqlalchemy import desc
 from flask_sqlalchemy import SQLAlchemy
-from anyway.parsers import infographics_data_cache_updater
 from anyway.parsers import timezones
 from anyway.models import NewsFlash
 from anyway.slack_accident_notifications import publish_notification
@@ -69,17 +68,15 @@ def insert_new_newsflash(self, newsflash: NewsFlash) -> None:
         self.__fill_na(newsflash)
         self.db.session.add(newsflash)
         self.db.session.commit()
-        infographics_data_cache_updater.add_news_flash_to_cache(newsflash)
         if os.environ.get("FLASK_ENV") == "production" and newsflash.accident:
             try:
                 DBAdapter.publish_notifications(newsflash)
             except Exception as e:
                 logging.error("publish notifications failed")
                 logging.error(e)
 
-
-    def get_newsflash_by_id(self, id):
-        return self.db.session.query(NewsFlash).filter(NewsFlash.id == id)
+    def get_newsflash_by_id(self, nid):
+        return self.db.session.query(NewsFlash).filter(NewsFlash.id == nid)
 
     def select_newsflash_where_source(self, source):
         return self.db.session.query(NewsFlash).filter(NewsFlash.source == source)

anyway/parsers/resolution_fields.py

@@ -0,0 +1,81 @@
+from typing import Union, List, Set, Container
+from anyway.backend_constants import BE_CONST
+
+
+class ResolutionFields:
+    RC = BE_CONST.ResolutionCategories
+    __required_fields = {
+        "מחוז": ["region_hebrew"],
+        "נפה": ["district_hebrew"],
+        "עיר": ["yishuv_name"],
+        "רחוב": ["yishuv_name", "street1_hebrew"],
+        "צומת עירוני": ["yishuv_name", "street1_hebrew", "street2_hebrew"],
+        "כביש בינעירוני": ["road1", "road_segment_id"],
+        "צומת בינעירוני": [
+            "non_urban_intersection",
+            "non_urban_intersection_hebrew",
+            "road1",
+            "road2",
+        ],
+        "אחר": [
+            "region_hebrew",
+            "district_hebrew",
+            "yishuv_name",
+            "street1_hebrew",
+            "street2_hebrew",
+            "non_urban_intersection_hebrew",
+            "road1",
+            "road2",
+            "road_segment_name",
+            "road_segment_id",
+        ],
+    }
+    __possible_fields = {
+        "מחוז": __required_fields[RC.REGION.value],
+        "נפה": __required_fields[RC.DISTRICT.value],
+        "עיר": __required_fields[RC.CITY.value],
+        "רחוב": __required_fields[RC.STREET.value],
+        "צומת עירוני": __required_fields[RC.URBAN_JUNCTION.value],
+        "כביש בינעירוני": ["road1", "road_segment_id", "road_segment_name"],
+        "צומת בינעירוני": __required_fields[RC.SUBURBAN_JUNCTION.value],
+        "אחר": __required_fields[RC.OTHER.value],
+    }
+    __all_fields = set()
+
+    @classmethod
+    def get_fields(cls, d: dict, res: Union[BE_CONST.ResolutionCategories, str]) -> List[str]:
+        if isinstance(res, BE_CONST.ResolutionCategories):
+            res = res.value
+        if res in d:
+            return d[res]
+        else:
+            raise ValueError(f"ResolutionFields:{res}: not a resolution")
+
+    @classmethod
+    def get_possible_fields(cls, res: Union[BE_CONST.ResolutionCategories, str]) -> List[str]:
+        return cls.get_fields(cls.__possible_fields, res)
+
+    @classmethod
+    def get_required_fields(cls, res: Union[BE_CONST.ResolutionCategories, str]) -> List[str]:
+        return cls.get_fields(cls.__required_fields, res)
+
+    @classmethod
+    def get_all_location_fields(cls) -> Set[str]:
+        if not cls.__all_fields:
+            for fields in cls.__possible_fields.values():
+                cls.__all_fields.update(fields)
+        return cls.__all_fields
+
+    @classmethod
+    def is_resolution_valid(cls, res: str) -> bool:
+        return res in cls.__possible_fields
+
+    @classmethod
+    def get_supported_resolution_of_fields(cls, fields: Container[str]) -> List[str]:
+        fields = set(fields)
+        res = list(
+            filter(
+                lambda r: set(cls.get_required_fields(r)) <= fields, BE_CONST.SUPPORTED_RESOLUTIONS
+            )
+        )
+        return res