diff --git a/anyway/globalmaptiles.py b/anyway/globalmaptiles.py index 4edf23a58..38a367e6c 100644 --- a/anyway/globalmaptiles.py +++ b/anyway/globalmaptiles.py @@ -215,12 +215,6 @@ def PixelsToTile(self, px, py): ty = int(math.ceil(py / float(self.tileSize)) - 1) return tx, ty - def PixelsToRaster(self, px, py, zoom): - "Move the origin of pixel coordinates to top-left corner" - - mapSize = self.tileSize << zoom - return px, mapSize - py - def MetersToTile(self, mx, my, zoom): "Returns tile for given mercator coordinates" @@ -249,13 +243,6 @@ def Resolution(self, zoom): # return (2 * math.pi * 6378137) / (self.tileSize * 2**zoom) return self.initialResolution / (2 ** zoom) - def ZoomForPixelSize(self, pixelSize): - "Maximal scaledown zoom of the pyramid closest to the pixelSize." - - for i in range(30): - if pixelSize > self.Resolution(i): - return i - 1 if i != 0 else 0 # We don't want to scale up - def GoogleTile(self, tx, ty, zoom): "Converts TMS tile coordinates to Google Tile coordinates" @@ -320,14 +307,6 @@ class GlobalGeodetic(object): def __init__(self, tileSize=256): self.tileSize = tileSize - def LatLonToPixels(self, lat, lon, zoom): - "Converts lat/lon to pixel coordinates in given zoom of the EPSG:4326 pyramid" - - res = 180 / 256.0 / 2 ** zoom - px = (180 + lat) / res - py = (90 + lon) / res - return px, py - def PixelsToTile(self, px, py): "Returns coordinates of the tile covering region in pixel coordinates" diff --git a/anyway/graphqlSchema.py b/anyway/graphqlSchema.py index e5023a3e3..a041ff030 100644 --- a/anyway/graphqlSchema.py +++ b/anyway/graphqlSchema.py @@ -20,9 +20,5 @@ class Query(graphene.ObjectType): node = relay.Node.Field() accident_markers_hebrew = graphene.List(AccidentMarkerHebrew) - def resolve_accident_markers_hebrew(self, info): - query = AccidentMarkerHebrew.get_query(info) - return query.all() - schema = graphene.Schema(query=Query) diff --git a/anyway/infographics_utils.py b/anyway/infographics_utils.py index d6f48b748..a9a717c1b 100755 --- a/anyway/infographics_utils.py +++ b/anyway/infographics_utils.py @@ -6,7 +6,6 @@ import traceback from typing import Optional, Dict, List, Type -from collections import defaultdict # noinspection PyProtectedMember from flask_babel import _ @@ -19,10 +18,9 @@ extract_news_flash_location, get_request_params_from_request_values, ) -from anyway.backend_constants import BE_CONST, AccidentType +from anyway.backend_constants import BE_CONST from anyway.models import NewsFlash, AccidentMarkerView from anyway.parsers import resolution_dict -from anyway.infographics_dictionaries import head_on_collisions_comparison_dict from anyway.parsers import infographics_data_cache_updater from anyway.widgets.widget import Widget, widgets_dict @@ -46,42 +44,10 @@ ITEMS = "items" -def get_widget_factories() -> List[Type[Widget]]: - """Returns list of callables that generate all widget instances""" - return list(widgets_dict.values()) - - def get_widget_class_by_name(name: str) -> Type[Widget]: return widgets_dict.get(name) -def sum_road_accidents_by_specific_type(road_data, field_name): - dict_merge = defaultdict(int) - dict_merge[field_name] = 0 - dict_merge[head_on_collisions_comparison_dict["others"]] = 0 - - for accident_data in road_data: - if accident_data["accident_type"] == field_name: - dict_merge[field_name] += accident_data["count"] - else: - dict_merge[head_on_collisions_comparison_dict["others"]] += accident_data["count"] - return dict_merge - - -def convert_roads_fatal_accidents_to_frontend_view(data_dict): - data_list = [] - for key, value in data_dict.items(): - # pylint: disable=no-member - if key == AccidentType.HEAD_ON_FRONTAL_COLLISION.value: - data_list.append( - {"desc": head_on_collisions_comparison_dict["head_to_head"], "count": value} - ) - else: - data_list.append({"desc": key, "count": value}) - - return data_list - - def generate_widgets_data(request_params: RequestParams) -> List[dict]: res = [] for w in widgets_dict.values(): diff --git a/anyway/models.py b/anyway/models.py index 54c51b827..aebf7c8a6 100755 --- a/anyway/models.py +++ b/anyway/models.py @@ -47,7 +47,6 @@ class UserMixin: from anyway import localization from anyway.backend_constants import BE_CONST, NewsflashLocationQualification from anyway.database import Base -from anyway.utilities import decode_hebrew try: from anyway.app_and_db import db @@ -223,8 +222,8 @@ class MarkerMixin(Point): def format_description(field, value): # if the field's value is a static localizable field, fetch it. if field in localization.get_supported_tables(): - value = decode_hebrew(localization.get_field(field, value)) - name = decode_hebrew(localization.get_field(field)) + value = localization.get_field(field, value) + name = localization.get_field(field) return "{0}: {1}".format(name, value) @@ -1211,24 +1210,6 @@ class SuburbanJunction(Base): nullable=True) roads = Column(postgresql.ARRAY(Integer(), dimensions=1), nullable=False) - @staticmethod - def get_hebrew_name_from_id(non_urban_intersection: int) -> str: - res = db.session.query(SuburbanJunction.non_urban_intersection_hebrew).filter( - SuburbanJunction.non_urban_intersection == non_urban_intersection).first() - if res is None: - raise ValueError(f"{non_urban_intersection}: could not find " - f"SuburbanJunction with that symbol") - return res.non_urban_intersection_hebrew - - @staticmethod - def get_id_from_hebrew_name(non_urban_intersection_hebrew: str) -> int: - res = db.session.query(SuburbanJunction.non_urban_intersection).filter( - SuburbanJunction.non_urban_intersection == non_urban_intersection_hebrew).first() - if res is None: - raise ValueError(f"{non_urban_intersection_hebrew}: could not find " - f"SuburbanJunction with that name") - return res.non_urban_intersection - @staticmethod def get_intersection_from_roads(roads: Set[int]) -> dict: if not all([isinstance(x, int) for x in roads]): @@ -2274,7 +2255,6 @@ def serialize(self): } - class ReportProblem(Base): __tablename__ = "report_problem" id = Column(BigInteger(), autoincrement=True, primary_key=True, index=True) @@ -2928,9 +2908,6 @@ class CasualtiesCosts(Base): year = Column(Integer()) data_source_hebrew = Column(String()) - def to_str(self): - return f"{self.id}:{self.injured_type}:{self.injuries_cost_k}" - class SchoolWithDescription2020(Base): __tablename__ = "schools_with_description2020" diff --git a/anyway/parsers/cbs/exceptions.py b/anyway/parsers/cbs/exceptions.py deleted file mode 100644 index cb95d0e22..000000000 --- a/anyway/parsers/cbs/exceptions.py +++ /dev/null @@ -1,6 +0,0 @@ -from anyway.exceptions import AnywayError - - -class CBSParsingFailed(AnywayError): - def __init__(self, message: str): - super().__init__(message=f"Exception occurred while loading the cbs data: {message}") diff --git a/anyway/parsers/cbs/executor.py b/anyway/parsers/cbs/executor.py index edc08eed8..345a98d6b 100644 --- a/anyway/parsers/cbs/executor.py +++ b/anyway/parsers/cbs/executor.py @@ -84,7 +84,6 @@ VehiclesView, VehicleMarkerView, ) -from anyway.parsers.cbs.exceptions import CBSParsingFailed from anyway.utilities import ItmToWGS84, time_delta, ImporterUI, truncate_tables, delete_all_rows_from_table, \ chunks, run_query_and_insert_to_table_in_chunks from anyway.db_views import VIEWS @@ -903,45 +902,6 @@ def add_suburban_junction_from_marker(marker: dict): add_suburban_junction(j) -def delete_invalid_entries(batch_size): - """ - deletes all markers in the database with null latitude or longitude - first deletes from tables Involved and Vehicle, then from table AccidentMarker - """ - - marker_ids_to_delete = ( - db.session.query(AccidentMarker.id) - .filter(or_((AccidentMarker.longitude == None), (AccidentMarker.latitude == None))) - .all() - ) - - marker_ids_to_delete = [acc_id[0] for acc_id in marker_ids_to_delete] - - logging.debug("There are " + str(len(marker_ids_to_delete)) + " invalid accident_ids to delete") - - for ids_chunk in chunks(marker_ids_to_delete, batch_size): - - logging.debug("Deleting a chunk of " + str(len(ids_chunk))) - - q = db.session.query(Involved).filter(Involved.accident_id.in_(ids_chunk)) - if q.all(): - logging.debug("deleting invalid entries from Involved") - q.delete(synchronize_session="fetch") - db.session.commit() - - q = db.session.query(Vehicle).filter(Vehicle.accident_id.in_(ids_chunk)) - if q.all(): - logging.debug("deleting invalid entries from Vehicle") - q.delete(synchronize_session="fetch") - db.session.commit() - - q = db.session.query(AccidentMarker).filter(AccidentMarker.id.in_(ids_chunk)) - if q.all(): - logging.debug("deleting invalid entries from AccidentMarker") - q.delete(synchronize_session="fetch") - db.session.commit() - - def delete_cbs_entries(start_year, batch_size): """ deletes all CBS markers (provider_code=1 or provider_code=3) in the database created in year and with provider code provider_code @@ -1252,5 +1212,5 @@ def main(batch_size, source, load_start_year=None): logging.debug("Finished Creating Hebrew DB Tables") except Exception as ex: print("Traceback: {0}".format(traceback.format_exc())) - raise CBSParsingFailed(message=str(ex)) + raise RuntimeError(f'Exception occurred while loading the cbs data: {ex}') # Todo - send an email that an exception occured diff --git a/anyway/parsers/infographics_data_cache_updater.py b/anyway/parsers/infographics_data_cache_updater.py index 9e9f96371..1e59c6332 100755 --- a/anyway/parsers/infographics_data_cache_updater.py +++ b/anyway/parsers/infographics_data_cache_updater.py @@ -84,52 +84,6 @@ def add_news_flash_to_cache(news_flash: NewsFlash): return False -def get_infographics_data_from_cache(news_flash_id, years_ago) -> Dict: - db_item = ( - db.session.query(InfographicsDataCache) - .filter(InfographicsDataCache.news_flash_id == news_flash_id) - .filter(InfographicsDataCache.years_ago == years_ago) - .first() - ) - logging.debug(f"retrieved from cache {type(db_item)}:{db_item}"[:70]) - db.session.commit() - try: - if db_item: - return json.loads(db_item.get_data()) - else: - return {} - except Exception as e: - logging.error( - f"Exception while extracting data from returned cache item flash_id:{news_flash_id},years:{years_ago})" - f"returned value {type(db_item)}" - f":cause:{e.__cause__}, class:{e.__class__}" - ) - return {} - - -def get_infographics_data_from_cache_by_road_segment(road_segment_id, years_ago) -> Dict: - db_item = ( - db.session.query(InfographicsRoadSegmentsDataCache) - .filter(InfographicsRoadSegmentsDataCache.road_segment_id == int(road_segment_id)) - .filter(InfographicsRoadSegmentsDataCache.years_ago == int(years_ago)) - .first() - ) - logging.debug(f"retrieved from cache {type(db_item)}:{db_item}"[:70]) - db.session.commit() - try: - if db_item: - return json.loads(db_item.get_data()) - else: - return {} - except Exception as e: - logging.error( - f"Exception while extracting data from returned cache item flash_id:{road_segment_id},years:{years_ago})" - f"returned value {type(db_item)}" - f":cause:{e.__cause__}, class:{e.__class__}" - ) - return {} - - def get_cache_retrieval_query(params: RequestParams): res = params.resolution loc = params.location_info diff --git a/anyway/parsers/location_extraction.py b/anyway/parsers/location_extraction.py index 809ee37e9..031c7f4d2 100644 --- a/anyway/parsers/location_extraction.py +++ b/anyway/parsers/location_extraction.py @@ -258,61 +258,6 @@ def set_accident_resolution(accident_row): logging.info("bug in accident resolution") -def reverse_geocode_extract(latitude, longitude): - """ - this method takes a latitude, longitude and returns a dict of the corresponding - location found on google maps (by that string), describing details of the location found and the geometry - :param latitude: latitude - :param longitude: longitude - :return: a dict containing data about the found location on google maps, with the keys: street, - road_no [road number], intersection, city, address, district and the geometry of the location. - """ - street = None - road_no = None - intersection = None - subdistrict = None - city = None - district = None - try: - gmaps = googlemaps.Client(key=secrets.get("GOOGLE_MAPS_KEY")) - geocode_result = gmaps.reverse_geocode((latitude, longitude)) - - # if we got no results, move to next iteration of location string - if not geocode_result: - return None - except Exception as _: - logging.info("exception in gmaps") - return None - # logging.info(geocode_result) - response = geocode_result[0] - geom = response["geometry"]["location"] - for item in response["address_components"]: - if "route" in item["types"]: - if item["short_name"].isdigit(): - road_no = int(item["short_name"]) - else: - street = item["long_name"] - elif "point_of_interest" in item["types"] or "intersection" in item["types"]: - intersection = item["long_name"] - elif "locality" in item["types"]: - city = item["long_name"] - elif "administrative_area_level_2" in item["types"]: - subdistrict = item["long_name"] - elif "administrative_area_level_1" in item["types"]: - district = item["long_name"] - address = response["formatted_address"] - return { - "street": street, - "road_no": road_no, - "intersection": intersection, - "city": city, - "address": address, - "subdistrict": subdistrict, - "district": district, - "geom": geom, - } - - def geocode_extract(location): """ this method takes a string representing location and a google maps key and returns a dict of the corresponding diff --git a/anyway/parsers/news_flash_db_adapter.py b/anyway/parsers/news_flash_db_adapter.py index a25e6195b..4ebecbc51 100644 --- a/anyway/parsers/news_flash_db_adapter.py +++ b/anyway/parsers/news_flash_db_adapter.py @@ -30,30 +30,6 @@ def execute(self, *args, **kwargs): def commit(self, *args, **kwargs): return self.db.session.commit(*args, **kwargs) - def recreate_table_for_location_extraction(self): - with self.db.session.begin(): - self.db.session.execute("""TRUNCATE cbs_locations""") - self.db.session.execute("""INSERT INTO cbs_locations - (SELECT ROW_NUMBER() OVER (ORDER BY road1) as id, LOCATIONS.* - FROM - (SELECT DISTINCT road1, - road2, - non_urban_intersection_hebrew, - yishuv_name, - street1_hebrew, - street2_hebrew, - district_hebrew, - region_hebrew, - road_segment_name, - longitude, - latitude - FROM markers_hebrew - WHERE (provider_code=1 - OR provider_code=3) - AND (longitude is not null - AND latitude is not null)) LOCATIONS)""" - ) - def get_markers_for_location_extraction(self): query_res = self.execute( """SELECT * FROM cbs_locations""" diff --git a/anyway/parsers/registered.py b/anyway/parsers/registered.py index 3764915b5..dd89dd8a9 100644 --- a/anyway/parsers/registered.py +++ b/anyway/parsers/registered.py @@ -6,7 +6,7 @@ import re from datetime import datetime from anyway.models import RegisteredVehicle, DeprecatedCity -from anyway.utilities import time_delta, CsvReader, ImporterUI, truncate_tables, decode_hebrew +from anyway.utilities import time_delta, CsvReader, ImporterUI, truncate_tables from anyway.app_and_db import db @@ -88,9 +88,9 @@ def row_parse(self, row): return { "year": self._report_year, - "name": decode_hebrew(name), + "name": name, "name_eng": row[COLUMN_CITY_NAME_ENG].strip(), - "search_name": decode_hebrew(search_name), + "search_name": search_name, "motorcycle": self.as_int(row[COLUMN_CITY_TOTAL_MOTORCYCLE]), "special": self.as_int(row[COLUMN_CITY_TOTAL_SPECIAL]), "taxi": self.as_int(row[COLUMN_CITY_TOTAL_TAXI]), diff --git a/anyway/parsers/twitter.py b/anyway/parsers/twitter.py index 7da373534..7928e9626 100644 --- a/anyway/parsers/twitter.py +++ b/anyway/parsers/twitter.py @@ -1,5 +1,3 @@ -import re - import tweepy from anyway import secrets @@ -32,15 +30,6 @@ def scrape(screen_name, latest_tweet_id=None, count=100): yield parse_tweet(tweet, screen_name) -def extract_accident_time(text): - # Currently unused - reg_exp = r"בשעה (\d{2}:\d{2})" - time_search = re.search(reg_exp, text) - if time_search: - return time_search.group(1) - return "" - - def parse_tweet(tweet, screen_name): return NewsFlash( link="https://twitter.com/{}/status/{}".format(screen_name, tweet["id_str"]), diff --git a/anyway/pymapcluster.py b/anyway/pymapcluster.py index fb723c3cc..e607a1428 100644 --- a/anyway/pymapcluster.py +++ b/anyway/pymapcluster.py @@ -1,5 +1,5 @@ ## -from math import cos, sin, atan2, sqrt +from math import sqrt from anyway import globalmaptiles as globaltiles @@ -7,31 +7,6 @@ ## -def center_geolocation(geolocations): - """ - Provide a relatively accurate center lat, lon returned as a list pair, given - a list of list pairs. - ex: in: geolocations = ((lat1,lon1), (lat2,lon2),) - out: (center_lat, center_lon) - """ - x = 0 - y = 0 - z = 0 - - for lat, lon in geolocations: - lat = float(lat) - lon = float(lon) - x += cos(lat) * cos(lon) - y += cos(lat) * sin(lon) - z += sin(lat) - - x = float(x / len(geolocations)) - y = float(y / len(geolocations)) - z = float(z / len(geolocations)) - - return (atan2(y, x), atan2(z, sqrt(x * x + y * y))) - - def latlng_to_zoompixels(mercator, lat, lng, zoom): mx, my = mercator.LatLonToMeters(lat, lng) pix = mercator.MetersToPixels(mx, my, zoom) @@ -74,7 +49,6 @@ def cluster_markers(mercator, latlngs, zoom, gridsize=50): break if not assigned: # Create new cluster for point - # TODO center_geolocation the center! centers.append(i) sizes.append(1) clusters.append(len(centers) - 1) @@ -97,13 +71,6 @@ def get_cluster_json(clust_marker, clust_size): } -def get_cluster_size(index, clusters): - from collections import Counter - - # TODO: don't call Counter for every cluster in the array - return Counter(clusters)[index] - - def calculate_clusters(markers, zoom, radius=50): centers, _, sizes = create_clusters_centers(markers, zoom, radius) json_clusts = [] diff --git a/anyway/utilities.py b/anyway/utilities.py index 7181e3d54..4d03b968e 100644 --- a/anyway/utilities.py +++ b/anyway/utilities.py @@ -3,7 +3,6 @@ import math import os import re -import sys import threading import typing from csv import DictReader @@ -71,20 +70,6 @@ def init_flask(): return app -class ProgressSpinner(object): - def __init__(self): - self.counter = 0 - self.chars = ["|", "/", "-", "\\"] - - def show(self): - """ - prints a rotating spinner - """ - current_char = self.counter % len(self.chars) - sys.stderr.write("\r%s" % self.chars[current_char]) - self.counter += 1 - - class CsvReader(object): """ loads and handles csv files @@ -162,17 +147,9 @@ def time_delta(since): ) -def decode_hebrew(s): - return s - - open_utf8 = partial(open, encoding="utf-8") -def row_to_dict(row): - return row._asdict() - - def fetch_first_and_every_nth_value_for_column(conn, column_to_fetch, n): sub_query = select([])\ .column(column_to_fetch)\ diff --git a/anyway/vehicle_type.py b/anyway/vehicle_type.py index 67f602791..9b229d745 100644 --- a/anyway/vehicle_type.py +++ b/anyway/vehicle_type.py @@ -1,7 +1,6 @@ from enum import Enum -from typing import List, Union +from typing import List import logging -import math try: from flask_babel import _ @@ -37,13 +36,6 @@ class VehicleType(Enum): TRUCK_3_5_TO_10 = 24 TRUCK_10_TO_12 = 25 - def get_categories(self) -> List[int]: - res = [] - for t in list(VehicleCategory): - if self in t.value: - res.append(t) - return res - def get_english_display_name(self): english_vehicle_type_display_names = { VehicleType.CAR: "private car", @@ -77,23 +69,6 @@ def get_english_display_name(self): logging.exception(f"VehicleType.get_display_name: {self}: no display string defined") return "no display name defined" - @staticmethod - def to_type_code(db_val: Union[float, int]) -> int: - """Values read from DB may arrive as float, and empty values come as nan""" - if isinstance(db_val, float): - if math.isnan(db_val): - return VehicleType.OTHER_AND_UNKNOWN.value - else: - return int(db_val) - elif isinstance(db_val, int): - return db_val - else: - logging.error( - f"VehicleType.fo_type_code: unknown value: {db_val}({type(db_val)})" - ". returning OTHER_AND_UNKNOWN" - ) - return VehicleType.OTHER_AND_UNKNOWN.value - VT = VehicleType diff --git a/anyway/widgets/widget.py b/anyway/widgets/widget.py index 59c2d58f0..86d826f84 100644 --- a/anyway/widgets/widget.py +++ b/anyway/widgets/widget.py @@ -35,12 +35,6 @@ def __init__(self, request_params: RequestParams): self.meta = {"widget_digest": self.widget_digest} self.information = "" - def get_name(self) -> str: - return self.name - - def get_rank(self) -> int: - return self.rank - @staticmethod def is_in_cache() -> bool: """Whether this widget is stored in the cache""" @@ -69,10 +63,6 @@ def localize_items(request_params: RequestParams, items: Dict) -> Dict: logging.error(f"Widget.localize_items: bad input (missing 'name' key):{items}") return items - @classmethod - def get_widget_files(cls) -> List[str]: - return cls.files - @staticmethod def calc_widget_digest(files: List[str]) -> str: h = hashlib.md5() diff --git a/tests/parsers/cbs/test_executor.py b/tests/parsers/cbs/test_executor.py index bd096e0d6..46f3a6cf6 100644 --- a/tests/parsers/cbs/test_executor.py +++ b/tests/parsers/cbs/test_executor.py @@ -2,7 +2,6 @@ import pytest -from anyway.parsers.cbs.exceptions import CBSParsingFailed from anyway.parsers.cbs.executor import main @pytest.fixture @@ -33,6 +32,5 @@ def test_cbs_parsing_failed_is_raised_when_something_bad_happens(monkeypatch): monkeypatch.setattr('anyway.parsers.cbs.executor.load_existing_streets', MagicMock(side_effect=Exception('something bad'))) - with pytest.raises(CBSParsingFailed, match='Exception occurred while loading the cbs data: something bad'): + with pytest.raises(RuntimeError, match='Exception occurred while loading the cbs data: something bad'): main(batch_size=MagicMock(), source=MagicMock()) -