From 69f674ecbe7894316ee822490f14e029fab0c845 Mon Sep 17 00:00:00 2001 From: hedingber Date: Wed, 18 Nov 2020 21:24:06 +0200 Subject: [PATCH 1/3] Add weather interpolator --- anyway/parsers/cbs/weather_interpolator.py | 213 ++++++++ anyway/parsers/cbs/weather_stations.xml | 598 +++++++++++++++++++++ main.py | 25 + tests/test_get_weather.py | 40 ++ 4 files changed, 876 insertions(+) create mode 100644 anyway/parsers/cbs/weather_interpolator.py create mode 100644 anyway/parsers/cbs/weather_stations.xml create mode 100644 tests/test_get_weather.py diff --git a/anyway/parsers/cbs/weather_interpolator.py b/anyway/parsers/cbs/weather_interpolator.py new file mode 100644 index 000000000..d3bc77b6d --- /dev/null +++ b/anyway/parsers/cbs/weather_interpolator.py @@ -0,0 +1,213 @@ +import datetime +import logging +import math +import collections +import random + +from lxml import etree + +WEATHER_HISTORY_WINDOW = 60 * 60 # sec +WEATHER_SAMPLING_INTERVAL = 10 * 60 # sec +WEATHER_STATION_XML = "anyway/parsers/cbs/weather_stations.xml" +DEFAULT_NUMBER_OF_INTERPOLATION_POINTS = 3 + + +def get_weather( + latitude, longitude, timestamp, interpolation_points=DEFAULT_NUMBER_OF_INTERPOLATION_POINTS +): + timestamp = datetime.datetime.fromisoformat(timestamp) + + # Phase I: find N closest weather stations to target coordinates (N==interpolation_points) + closest_stations = get_closest_stations( + WEATHER_STATION_XML, latitude, longitude, interpolation_points + ) + logging.debug(f"Closest stations: {str(closest_stations)}") + + # Phase II: find weather history for each station + for idx in range(len(closest_stations)): + closest_stations[idx]["weather"] = get_weather_at_station( + closest_stations[idx]["id"], timestamp + ) + + # Phase III: Calculate time weighted weather for each station based on weather history + for idx in range(len(closest_stations)): + closest_stations[idx][ + "time_weighted_weather_parameters" + ] = weight_weather_parameters_in_time_domain(closest_stations[idx]["weather"]) + + # Phase IV: Create distance/weather_param_value tuples for spatial interpolation + weather_parameters_for_spatial_interpolation = {} + for station in closest_stations: + for parameter_name, parameter_value in station["time_weighted_weather_parameters"].items(): + weather_parameters_for_spatial_interpolation.setdefault(parameter_name, []).append( + (station["distance"], parameter_value) + ) + + # Phase V: Find weather parameters at target location using + # inverse distance weighted interpolation on station weather data + weather_at_target = {} + for ( + weather_parameter, + distance_value_tuples, + ) in weather_parameters_for_spatial_interpolation.items(): + weather_at_target[weather_parameter] = perform_inverse_distance_weighted_interpolation( + distance_value_tuples + ) + + logging.debug(f"Weather at target: {str(weather_at_target)}") + + return weather_at_target + + +def parse_xml(xml_file_location): + """ + Parse the xml + """ + with open(xml_file_location) as fobj: + xml = fobj.read().encode() + + weather_stations_locations = [] + root = etree.fromstring(xml) # pylint: disable=c-extension-no-member + for appt in root.getchildren(): + weather_station_parameters = {} + for elem in appt.getchildren(): + if elem.text: + if elem.tag in ["longitude", "latitude", "altitude"]: + weather_station_parameters[elem.tag] = float(elem.text) + elif elem.tag in ["id"]: + weather_station_parameters[elem.tag] = int(elem.text) + else: + weather_station_parameters[elem.tag] = elem.text + + weather_stations_locations.append(weather_station_parameters) + + return weather_stations_locations + + +def get_distance(origin, destination): + """ + Calculate the Haversine distance. + + Parameters + ---------- + origin : tuple of float + (lat, long) + destination : tuple of float + (lat, long) + + Returns + ------- + distance_in_km : float + + Examples + -------- + >>> origin = (48.1372, 11.5756) # Munich + >>> destination = (52.5186, 13.4083) # Berlin + >>> round(get_distance(origin, destination), 1) + 504.2 + """ + lat1, lon1 = origin + lat2, lon2 = destination + radius = 6371 # km + + dlat = math.radians(lat2 - lat1) + dlon = math.radians(lon2 - lon1) + a = math.sin(dlat / 2) * math.sin(dlat / 2) + math.cos(math.radians(lat1)) * math.cos( + math.radians(lat2) + ) * math.sin(dlon / 2) * math.sin(dlon / 2) + c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) + d = radius * c + + return d + + +def get_closest_stations(weather_stations_locations_xml, latitude, longitude, num_of_stations): + + # Load XML file with weather stations' Latitude/Longitude coordinates + weather_stations_locations = parse_xml(weather_stations_locations_xml) + + # Calculate the distance of target location from each weather station + target_location = (latitude, longitude) + distances_to_weather_stations = {} + for weather_station in weather_stations_locations: + distance_to_station = get_distance( + target_location, (weather_station["latitude"], weather_station["longitude"]) + ) + distances_to_weather_stations[distance_to_station] = weather_station + + # Sort the results + sorted_distances = collections.OrderedDict(sorted(distances_to_weather_stations.items())) + + # Return response with N closest stations (N==num_of_stations) + idx = 0 + closest_stations = [] + for _distance, weather_station in sorted_distances.items(): + weather_station.update({"distance": _distance}) + closest_stations.append( + { + "id": weather_station["id"], + "name": weather_station["name"], + # to avoid "divide by zero" situations, round 0 to 0.01 + "distance": max(0.01, _distance), + } + ) + + idx += 1 + if idx >= num_of_stations: + break + + return closest_stations + + +def get_weather_at_station(station_id, timestamp): + # TODO: replace this mock with actual request to meteorological service API once we have the TOKEN + # https://ims.gov.il/node/87 + + results = [] + + # The API should return response with samples in 10min intervals + idx = station_id + for delta_sec in range(0, WEATHER_HISTORY_WINDOW, WEATHER_SAMPLING_INTERVAL): + results.append( + { + "timestamp": timestamp - datetime.timedelta(seconds=delta_sec), + "weather_parameters": { + "temperature": random.uniform(0, 45), + "rain": idx, + }, + } + ) + + idx += 2 + + return results + + +def weight_weather_parameters_in_time_domain(weather_samples): + weights = [] + for idx in range(len(weather_samples) - 1): + weights.insert(0, 1.0 / (2 ** (len(weather_samples) - idx))) + weights.insert(0, 1 - sum(weights)) + + time_weighted_parameters = {} + for idx, weather_sample in enumerate(weather_samples): + for parameter_name, parameter_value in weather_sample["weather_parameters"].items(): + if parameter_name in ["rain"]: + time_weighted_parameters[parameter_name] = ( + time_weighted_parameters.get(parameter_name, 0) + parameter_value * weights[idx] + ) + + return time_weighted_parameters + + +def perform_inverse_distance_weighted_interpolation(distance_value_tuples): + + # https://math.stackexchange.com/questions/1336386/weighted-average-distance-between-3-or-more-points + # https://stackoverflow.com/a/3119544 + numerator = 0 + denominator = 0 + for distance, value in distance_value_tuples: + numerator += value / distance + denominator += 1 / distance + + return numerator / denominator diff --git a/anyway/parsers/cbs/weather_stations.xml b/anyway/parsers/cbs/weather_stations.xml new file mode 100644 index 000000000..e118cd9f3 --- /dev/null +++ b/anyway/parsers/cbs/weather_stations.xml @@ -0,0 +1,598 @@ + + + + 1 + KEFAR GILADI + 35.5696 + 33.2404 + 365 + + + 2 + DAFNA + 35.6350 + 33.2277 + 135 + + + 3 + KEFAR BLUM + 35.6133 + 33.1716 + 75 + + + 4 + MEROM GOLAN PICMAN + 35.8045 + 33.1288 + 945 + + + 5 + ROSH HANIQRA + 35.1079 + 33.0806 + 10 + + + 6 + ELON + 35.2173 + 33.0653 + 300 + + + 7 + AYYELET HASHAHAR + 35.5735 + 33.0244 + 170 + + + 8 + SHAVE ZIYYON + 35.0923 + 32.9836 + 5 + + + 9 + ZEFAT HAR KENAAN + 35.5070 + 32.9800 + 936 + + + 10 + HARASHIM + 35.3287 + 32.9560 + 830 + + + 11 + AMMIAD + 35.5135 + 32.9150 + 215 + + + 12 + GAMLA + 35.7487 + 32.9052 + 405 + + + 13 + ESHHAR + 35.3015 + 32.8844 + 370 + + + 14 + KEFAR NAHUM + 35.5792 + 32.8835 + -200 + + + 15 + BET ZAYDA + 35.6504 + 32.8800 + -200 + + + 16 + DEIR HANNA + 35.3741 + 32.8621 + 280 + + + 17 + AFEQ + 35.1123 + 32.8466 + 10 + + + 18 + HAIFA PORT + 34.9977 + 32.8283 + 10 + + + 19 + LEV KINERET + 35.6053 + 32.8214 + -213 + + + 20 + AVNE ETAN + 35.7622 + 32.8174 + 375 + + + 21 + HAIFA REFINERIES + 35.0548 + 32.8034 + 5 + + + 22 + HAIFA TECHNION + 35.0223 + 32.7736 + 245 + + + 23 + HAIFA UNIVERSITY + 35.0208 + 32.7611 + 475 + + + 24 + NEWE YAAR + 35.1784 + 32.7078 + 115 + + + 25 + TAVOR KADOORIE + 35.4069 + 32.7053 + 145 + + + 26 + ZEMAH + 35.5839 + 32.7024 + -200 + + + 27 + YAVNEEL + 35.5101 + 32.6978 + 0 + + + 28 + MASSADA + 35.6008 + 32.6833 + -200 + + + 29 + EN KARMEL + 34.9594 + 32.6808 + 25 + + + 30 + MERHAVYA + 35.3075 + 32.6028 + 60 + + + 31 + EN HASHOFET + 35.0964 + 32.6028 + 265 + + + 32 + AFULA NIR HAEMEQ + 35.2769 + 32.5960 + 60 + + + 33 + ZIKHRON YAAQOV + 34.9546 + 32.5724 + 175 + + + 34 + GALED + 35.0725 + 32.5564 + 180 + + + 35 + TEL YOSEF + 35.3945 + 32.5462 + -65 + + + 36 + MAALE GILBOA + 35.4150 + 32.4810 + 495 + + + 37 + HADERA PORT + 34.8815 + 32.4732 + 5 + + + 38 + EDEN FARM + 35.4888 + 32.4679 + -120 + + + 39 + SEDE ELIYYAHU + 35.5106 + 32.4390 + -185 + + + 40 + EN HAHORESH + 34.9376 + 32.3877 + 15 + + + 41 + QARNE SHOMERON + 35.0959 + 32.1752 + 330 + + + 42 + ITAMAR + 35.3513 + 32.1598 + 820 + + + 43 + HAKFAR HAYAROK + 34.8324 + 32.1300 + 65 + + + 44 + SHAARE TIQWA + 35.0254 + 32.1257 + 195 + + + 45 + ARIEL + 35.2114 + 32.1056 + 685 + + + 46 + TEL AVIV COAST + 34.7588 + 32.0580 + 10 + + + 47 + BET DAGAN + 34.8138 + 32.0073 + 31 + + + 48 + GILGAL + 35.4509 + 31.9973 + -255 + + + 49 + HAR HARASHA + 35.1499 + 31.9449 + 770 + + + 50 + ASHDOD PORT + 34.6377 + 31.8342 + 5 + + + 51 + NAHSHON + 34.9616 + 31.8341 + 180 + + + 52 + QEVUZAT YAVNE + 34.7244 + 31.8166 + 50 + + + 53 + BET HAARAVA + 35.5010 + 31.8052 + -330 + + + 54 + HAFEZ HAYYIM + 34.8050 + 31.7910 + 80 + + + 55 + ZOVA + 35.1258 + 31.7878 + 710 + + + 56 + JERUSALEM CENTRE + 35.2217 + 31.7806 + 810 + + + 57 + MAALE ADUMMIM + 35.2961 + 31.7740 + 490 + + + 58 + JERUSALEM GIVAT RAM + 35.1973 + 31.7704 + 770 + + + 59 + NIZZAN + 34.6348 + 31.7319 + 30 + + + 60 + BEIT JIMAL + 34.9762 + 31.7248 + 355 + + + 61 + NETIV HALAMED HE + 34.9744 + 31.6898 + 275 + + + 62 + ROSH ZURIM + 35.1233 + 31.6644 + 950 + + + 63 + NEGBA + 34.6798 + 31.6585 + 95 + + + 64 + ASHQELON PORT + 34.5215 + 31.6394 + 5 + + + 65 + GAT + 34.7913 + 31.6303 + 140 + + + 66 + METZOKE DRAGOT + 35.3916 + 31.5881 + 20 + + + 67 + DOROT + 34.6480 + 31.5036 + 115 + + + 68 + EN GEDI + 35.3871 + 31.4200 + -415 + + + 69 + LAHAV + 34.8729 + 31.3812 + 460 + + + 70 + SHANI + 35.0662 + 31.3568 + 700 + + + 71 + BESOR FARM + 34.3894 + 31.2716 + 110 + + + 72 + BEER SHEVA + 34.7995 + 31.2515 + 279 + + + 73 + ARAD + 35.1855 + 31.2500 + 564 + + + 74 + NEVATIM + 34.9227 + 31.2050 + 350 + + + 75 + ZOMET HANEGEV + 34.8513 + 31.0708 + 360 + + + 76 + SEDOM + 35.3919 + 31.0306 + -388 + + + 77 + SEDE BOQER + 34.7950 + 30.8702 + 475 + + + 78 + EZUZ + 34.4715 + 30.7911 + 335 + + + 79 + AVDAT + 34.7712 + 30.7877 + 555 + + + 80 + HAZEVA + 35.2389 + 30.7787 + -135 + + + 81 + MIZPE RAMON + 34.8046 + 30.6101 + 845 + + + 82 + PARAN + 35.1479 + 30.3655 + 100 + + + 83 + NEOT SMADAR + 35.0233 + 30.0480 + 400 + + + 84 + YOTVATA + 35.0771 + 29.8851 + 70 + + + 85 + ELAT + 34.9520 + 29.5526 + 22 + + diff --git a/main.py b/main.py index de9e667b9..1f8c52a74 100755 --- a/main.py +++ b/main.py @@ -410,5 +410,30 @@ def update_casualties_costs(filename): return parse(filename) +@process.command() +@click.option("--latitude", type=float) +@click.option("--longitude", type=float) +@click.option("--interpolation_points", type=int, default=3) +@click.option( + "--timestamp", + type=str, + help="ISO formatted timestamp: 2011-11-04T00:05:23 / 2011-11-04 00:05:23.283 / 2011-11-04 00:05:23.283" +) +def get_weather( + latitude, + longitude, + interpolation_points, + timestamp +): + from anyway.parsers.cbs.weather_interpolator import get_weather + + return get_weather( + latitude=latitude, + longitude=longitude, + interpolation_points=interpolation_points, + timestamp=timestamp, + ) + + if __name__ == "__main__": cli(sys.argv[1:]) # pylint: disable=too-many-function-args diff --git a/tests/test_get_weather.py b/tests/test_get_weather.py new file mode 100644 index 000000000..f4c8e5d4d --- /dev/null +++ b/tests/test_get_weather.py @@ -0,0 +1,40 @@ +import anyway.parsers.cbs.weather_interpolator as weather_interpolator + + +class TestWeatherData: + + def test_get_weather_at_weather_station_location(self): + """ + Target coordinates are very close to TEL-AVIV weather station + So rain values at that station should be weighed at ~1 and values from other stations should be weighted at ~0 + """ + + rain_in_tel_aviv = list(range(46, 58, 2)) + time_weighted_value = 0 + for idx in range(len(self._get_weights())): + time_weighted_value += rain_in_tel_aviv[idx] * self._get_weights()[idx] + + weather_data = weather_interpolator.get_weather(32.0580, 34.7588, "2011-11-04T00:05:23", 3) + assert abs(weather_data["rain"] - time_weighted_value) < 0.01 + + def test_get_weather_at_equal_distance_from_3_weather_stations(self): + """ + Target coordinates are at almost the same distance from 3 closest weather stations + So rain values should be weighted equally + """ + + station_ids = [8, 5, 18] + time_weighted_value_values = [0, 0, 0] + + for station_idx, station_id in enumerate(station_ids): + rain_values = list(range(station_id, station_id+12, 2)) + for idx in range(len(self._get_weights())): + time_weighted_value_values[station_idx] += rain_values[idx] * self._get_weights()[idx] + + expected_rain = sum(time_weighted_value_values) / len(time_weighted_value_values) + + weather_data = weather_interpolator.get_weather(33.0580, 34.7588, "2011-11-04T00:05:23", 3) + assert abs(weather_data["rain"] - expected_rain) < 0.2 + + def _get_weights(self): + return [0.515625, 0.25, 0.125, 0.0625, 0.03125, 0.015625] From b56a8e9879132d8f71b67d4bb48b88ed8917034e Mon Sep 17 00:00:00 2001 From: hedingber Date: Wed, 18 Nov 2020 21:26:30 +0200 Subject: [PATCH 2/3] Add DB support for weather data + backfill logic --- .../5f15493966f7_adding_weather_data.py | 653 ++++++++++++++++++ anyway/db_views.py | 2 + anyway/models.py | 44 ++ anyway/parsers/cbs/executor.py | 3 + anyway/parsers/cbs/weather_data.py | 48 ++ main.py | 14 + tests/test_weather_data.py | 89 +++ 7 files changed, 853 insertions(+) create mode 100644 alembic/versions/5f15493966f7_adding_weather_data.py create mode 100644 anyway/parsers/cbs/weather_data.py create mode 100644 tests/test_weather_data.py diff --git a/alembic/versions/5f15493966f7_adding_weather_data.py b/alembic/versions/5f15493966f7_adding_weather_data.py new file mode 100644 index 000000000..e73285e0d --- /dev/null +++ b/alembic/versions/5f15493966f7_adding_weather_data.py @@ -0,0 +1,653 @@ +"""Adding weather data + +Revision ID: 5f15493966f7 +Revises: 9687ef04f99d +Create Date: 2020-10-20 21:30:51.699025 + +""" + +# revision identifiers, used by Alembic. +revision = '5f15493966f7' +down_revision = '9687ef04f99d' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +MARKERS_HEBREW_VIEW_WITH_WEATHER = """SELECT markers.id, + markers.provider_and_id, + markers.provider_code, + provider_code.provider_code_hebrew, + markers.file_type_police, + markers.accident_type, + accident_type.accident_type_hebrew, + markers.accident_severity, + accident_severity.accident_severity_hebrew, + markers.created as accident_timestamp, + markers.location_accuracy, + location_accuracy.location_accuracy_hebrew, + markers.road_type, + road_type.road_type_hebrew, + markers.road_shape, + road_shape.road_shape_hebrew, + markers.day_type, + day_type.day_type_hebrew, + markers.police_unit, + police_unit.police_unit_hebrew, + markers.one_lane, + one_lane.one_lane_hebrew, + markers.multi_lane, + multi_lane.multi_lane_hebrew, + markers.speed_limit, + speed_limit.speed_limit_hebrew, + markers.road_intactness, + road_intactness.road_intactness_hebrew, + markers.road_width, + road_width.road_width_hebrew, + markers.road_sign, + road_sign.road_sign_hebrew, + markers.road_light, + road_light.road_light_hebrew, + markers.road_control, + road_control.road_control_hebrew, + markers.weather, + weather.weather_hebrew, + markers.road_surface, + road_surface.road_surface_hebrew, + markers.road_object, + road_object.road_object_hebrew, + markers.object_distance, + object_distance.object_distance_hebrew, + markers.didnt_cross, + didnt_cross.didnt_cross_hebrew, + markers.cross_mode, + cross_mode.cross_mode_hebrew, + markers.cross_location, + cross_location.cross_location_hebrew, + markers.cross_direction, + cross_direction.cross_direction_hebrew, + markers.road1, + markers.road2, + markers.km, + markers.km_raw, + markers.km_accurate, + accident_weathers.rain_rate as accident_rain_rate, + road_segments.segment_id as road_segment_id, + road_segments.segment as road_segment_number, + road_segments.from_name || ' - ' || road_segments.to_name as road_segment_name, + road_segments.from_km as road_segment_from_km, + road_segments.to_km as road_segment_to_km, + road_segments.to_km - road_segments.from_km as road_segment_length_km, + markers.yishuv_symbol, + markers.yishuv_name, + markers.geo_area, + geo_area.geo_area_hebrew, + markers.day_night, + day_night.day_night_hebrew, + markers.day_in_week, + day_in_week.day_in_week_hebrew, + markers.traffic_light, + traffic_light.traffic_light_hebrew, + markers.region, + region.region_hebrew, + markers.district, + district.district_hebrew, + markers.natural_area, + natural_area.natural_area_hebrew, + markers.municipal_status, + municipal_status.municipal_status_hebrew, + markers.yishuv_shape, + yishuv_shape.yishuv_shape_hebrew, + markers.street1, + markers.street1_hebrew, + markers.street2, + markers.street2_hebrew, + markers.house_number, + markers.non_urban_intersection, + markers.non_urban_intersection_hebrew, + markers.non_urban_intersection_by_junction_number, + markers.urban_intersection, + markers.accident_year, + markers.accident_month, + markers.accident_day, + markers.accident_hour_raw, + accident_hour_raw.accident_hour_raw_hebrew, + markers.accident_hour, + markers.accident_minute, + markers.geom, + markers.longitude, + markers.latitude, + markers.x, + markers.y + FROM markers + LEFT JOIN road_segments on (markers.road1 = road_segments.road) AND (road_segments.from_km <= markers.km / 10) AND (markers.km / 10 < road_segments.to_km) + LEFT JOIN accident_type ON markers.accident_type = accident_type.id AND markers.accident_year = accident_type.year AND markers.provider_code = accident_type.provider_code + LEFT JOIN accident_severity ON markers.accident_severity = accident_severity.id AND markers.accident_year = accident_severity.year AND markers.provider_code = accident_severity.provider_code + LEFT JOIN location_accuracy ON markers.location_accuracy = location_accuracy.id AND markers.accident_year = location_accuracy.year AND markers.provider_code = location_accuracy.provider_code + LEFT JOIN road_type ON markers.road_type = road_type.id AND markers.accident_year = road_type.year AND markers.provider_code = road_type.provider_code + LEFT JOIN road_shape ON markers.road_shape = road_shape.id AND markers.accident_year = road_shape.year AND markers.provider_code = road_shape.provider_code + LEFT JOIN day_type ON markers.day_type = day_type.id AND markers.accident_year = day_type.year AND markers.provider_code = day_type.provider_code + LEFT JOIN police_unit ON markers.police_unit = police_unit.id AND markers.accident_year = police_unit.year AND markers.provider_code = police_unit.provider_code + LEFT JOIN one_lane ON markers.one_lane = one_lane.id AND markers.accident_year = one_lane.year AND markers.provider_code = one_lane.provider_code + LEFT JOIN multi_lane ON markers.multi_lane = multi_lane.id AND markers.accident_year = multi_lane.year AND markers.provider_code = multi_lane.provider_code + LEFT JOIN speed_limit ON markers.speed_limit = speed_limit.id AND markers.accident_year = speed_limit.year AND markers.provider_code = speed_limit.provider_code + LEFT JOIN road_intactness ON markers.road_intactness = road_intactness.id AND markers.accident_year = road_intactness.year AND markers.provider_code = road_intactness.provider_code + LEFT JOIN road_width ON markers.road_width = road_width.id AND markers.accident_year = road_width.year AND markers.provider_code = road_width.provider_code + LEFT JOIN road_sign ON markers.road_sign = road_sign.id AND markers.accident_year = road_sign.year AND markers.provider_code = road_sign.provider_code + LEFT JOIN road_light ON markers.road_light = road_light.id AND markers.accident_year = road_light.year AND markers.provider_code = road_light.provider_code + LEFT JOIN road_control ON markers.road_control = road_control.id AND markers.accident_year = road_control.year AND markers.provider_code = road_control.provider_code + LEFT JOIN weather ON markers.weather = weather.id AND markers.accident_year = weather.year AND markers.provider_code = weather.provider_code + LEFT JOIN accident_weathers ON accident_weathers.provider_code = markers.provider_code AND accident_weathers.accident_id = markers.id AND accident_weathers.accident_year = markers.accident_year + LEFT JOIN road_surface ON markers.road_surface = road_surface.id AND markers.accident_year = road_surface.year AND markers.provider_code = road_surface.provider_code + LEFT JOIN road_object ON markers.road_object = road_object.id AND markers.accident_year = road_object.year AND markers.provider_code = road_object.provider_code + LEFT JOIN object_distance ON markers.object_distance = object_distance.id AND markers.accident_year = object_distance.year AND markers.provider_code = object_distance.provider_code + LEFT JOIN didnt_cross ON markers.didnt_cross = didnt_cross.id AND markers.accident_year = didnt_cross.year AND markers.provider_code = didnt_cross.provider_code + LEFT JOIN cross_mode ON markers.cross_mode = cross_mode.id AND markers.accident_year = cross_mode.year AND markers.provider_code = cross_mode.provider_code + LEFT JOIN cross_location ON markers.cross_location = cross_location.id AND markers.accident_year = cross_location.year AND markers.provider_code = cross_location.provider_code + LEFT JOIN cross_direction ON markers.cross_direction = cross_direction.id AND markers.accident_year = cross_direction.year AND markers.provider_code = cross_direction.provider_code + LEFT JOIN geo_area ON markers.geo_area = geo_area.id AND markers.accident_year = geo_area.year AND markers.provider_code = geo_area.provider_code + LEFT JOIN day_night ON markers.day_night = day_night.id AND markers.accident_year = day_night.year AND markers.provider_code = day_night.provider_code + LEFT JOIN day_in_week ON markers.day_in_week = day_in_week.id AND markers.accident_year = day_in_week.year AND markers.provider_code = day_in_week.provider_code + LEFT JOIN traffic_light ON markers.traffic_light = traffic_light.id AND markers.accident_year = traffic_light.year AND markers.provider_code = traffic_light.provider_code + LEFT JOIN region ON markers.region = region.id AND markers.accident_year = region.year AND markers.provider_code = region.provider_code + LEFT JOIN district ON markers.district = district.id AND markers.accident_year = district.year AND markers.provider_code = district.provider_code + LEFT JOIN natural_area ON markers.natural_area = natural_area.id AND markers.accident_year = natural_area.year AND markers.provider_code = natural_area.provider_code + LEFT JOIN municipal_status ON markers.municipal_status = municipal_status.id AND markers.accident_year = municipal_status.year AND markers.provider_code = municipal_status.provider_code + LEFT JOIN yishuv_shape ON markers.yishuv_shape = yishuv_shape.id AND markers.accident_year = yishuv_shape.year AND markers.provider_code = yishuv_shape.provider_code + LEFT JOIN accident_hour_raw ON markers.accident_hour_raw = accident_hour_raw.id AND markers.accident_year = accident_hour_raw.year AND markers.provider_code = accident_hour_raw.provider_code + LEFT JOIN provider_code ON markers.provider_code = provider_code.id;""" + + +MARKERS_HEBREW_VIEW_WITHOUT_WEATHER = """SELECT markers.id, + markers.provider_and_id, + markers.provider_code, + provider_code.provider_code_hebrew, + markers.file_type_police, + markers.accident_type, + accident_type.accident_type_hebrew, + markers.accident_severity, + accident_severity.accident_severity_hebrew, + markers.created as accident_timestamp, + markers.location_accuracy, + location_accuracy.location_accuracy_hebrew, + markers.road_type, + road_type.road_type_hebrew, + markers.road_shape, + road_shape.road_shape_hebrew, + markers.day_type, + day_type.day_type_hebrew, + markers.police_unit, + police_unit.police_unit_hebrew, + markers.one_lane, + one_lane.one_lane_hebrew, + markers.multi_lane, + multi_lane.multi_lane_hebrew, + markers.speed_limit, + speed_limit.speed_limit_hebrew, + markers.road_intactness, + road_intactness.road_intactness_hebrew, + markers.road_width, + road_width.road_width_hebrew, + markers.road_sign, + road_sign.road_sign_hebrew, + markers.road_light, + road_light.road_light_hebrew, + markers.road_control, + road_control.road_control_hebrew, + markers.weather, + weather.weather_hebrew, + markers.road_surface, + road_surface.road_surface_hebrew, + markers.road_object, + road_object.road_object_hebrew, + markers.object_distance, + object_distance.object_distance_hebrew, + markers.didnt_cross, + didnt_cross.didnt_cross_hebrew, + markers.cross_mode, + cross_mode.cross_mode_hebrew, + markers.cross_location, + cross_location.cross_location_hebrew, + markers.cross_direction, + cross_direction.cross_direction_hebrew, + markers.road1, + markers.road2, + markers.km, + markers.km_raw, + markers.km_accurate, + road_segments.segment_id as road_segment_id, + road_segments.segment as road_segment_number, + road_segments.from_name || ' - ' || road_segments.to_name as road_segment_name, + road_segments.from_km as road_segment_from_km, + road_segments.to_km as road_segment_to_km, + road_segments.to_km - road_segments.from_km as road_segment_length_km, + markers.yishuv_symbol, + markers.yishuv_name, + markers.geo_area, + geo_area.geo_area_hebrew, + markers.day_night, + day_night.day_night_hebrew, + markers.day_in_week, + day_in_week.day_in_week_hebrew, + markers.traffic_light, + traffic_light.traffic_light_hebrew, + markers.region, + region.region_hebrew, + markers.district, + district.district_hebrew, + markers.natural_area, + natural_area.natural_area_hebrew, + markers.municipal_status, + municipal_status.municipal_status_hebrew, + markers.yishuv_shape, + yishuv_shape.yishuv_shape_hebrew, + markers.street1, + markers.street1_hebrew, + markers.street2, + markers.street2_hebrew, + markers.house_number, + markers.non_urban_intersection, + markers.non_urban_intersection_hebrew, + markers.non_urban_intersection_by_junction_number, + markers.urban_intersection, + markers.accident_year, + markers.accident_month, + markers.accident_day, + markers.accident_hour_raw, + accident_hour_raw.accident_hour_raw_hebrew, + markers.accident_hour, + markers.accident_minute, + markers.geom, + markers.longitude, + markers.latitude, + markers.x, + markers.y + FROM markers + LEFT JOIN road_segments on (markers.road1 = road_segments.road) AND (road_segments.from_km <= markers.km / 10) AND (markers.km / 10 < road_segments.to_km) + LEFT JOIN accident_type ON markers.accident_type = accident_type.id AND markers.accident_year = accident_type.year AND markers.provider_code = accident_type.provider_code + LEFT JOIN accident_severity ON markers.accident_severity = accident_severity.id AND markers.accident_year = accident_severity.year AND markers.provider_code = accident_severity.provider_code + LEFT JOIN location_accuracy ON markers.location_accuracy = location_accuracy.id AND markers.accident_year = location_accuracy.year AND markers.provider_code = location_accuracy.provider_code + LEFT JOIN road_type ON markers.road_type = road_type.id AND markers.accident_year = road_type.year AND markers.provider_code = road_type.provider_code + LEFT JOIN road_shape ON markers.road_shape = road_shape.id AND markers.accident_year = road_shape.year AND markers.provider_code = road_shape.provider_code + LEFT JOIN day_type ON markers.day_type = day_type.id AND markers.accident_year = day_type.year AND markers.provider_code = day_type.provider_code + LEFT JOIN police_unit ON markers.police_unit = police_unit.id AND markers.accident_year = police_unit.year AND markers.provider_code = police_unit.provider_code + LEFT JOIN one_lane ON markers.one_lane = one_lane.id AND markers.accident_year = one_lane.year AND markers.provider_code = one_lane.provider_code + LEFT JOIN multi_lane ON markers.multi_lane = multi_lane.id AND markers.accident_year = multi_lane.year AND markers.provider_code = multi_lane.provider_code + LEFT JOIN speed_limit ON markers.speed_limit = speed_limit.id AND markers.accident_year = speed_limit.year AND markers.provider_code = speed_limit.provider_code + LEFT JOIN road_intactness ON markers.road_intactness = road_intactness.id AND markers.accident_year = road_intactness.year AND markers.provider_code = road_intactness.provider_code + LEFT JOIN road_width ON markers.road_width = road_width.id AND markers.accident_year = road_width.year AND markers.provider_code = road_width.provider_code + LEFT JOIN road_sign ON markers.road_sign = road_sign.id AND markers.accident_year = road_sign.year AND markers.provider_code = road_sign.provider_code + LEFT JOIN road_light ON markers.road_light = road_light.id AND markers.accident_year = road_light.year AND markers.provider_code = road_light.provider_code + LEFT JOIN road_control ON markers.road_control = road_control.id AND markers.accident_year = road_control.year AND markers.provider_code = road_control.provider_code + LEFT JOIN weather ON markers.weather = weather.id AND markers.accident_year = weather.year AND markers.provider_code = weather.provider_code + LEFT JOIN road_surface ON markers.road_surface = road_surface.id AND markers.accident_year = road_surface.year AND markers.provider_code = road_surface.provider_code + LEFT JOIN road_object ON markers.road_object = road_object.id AND markers.accident_year = road_object.year AND markers.provider_code = road_object.provider_code + LEFT JOIN object_distance ON markers.object_distance = object_distance.id AND markers.accident_year = object_distance.year AND markers.provider_code = object_distance.provider_code + LEFT JOIN didnt_cross ON markers.didnt_cross = didnt_cross.id AND markers.accident_year = didnt_cross.year AND markers.provider_code = didnt_cross.provider_code + LEFT JOIN cross_mode ON markers.cross_mode = cross_mode.id AND markers.accident_year = cross_mode.year AND markers.provider_code = cross_mode.provider_code + LEFT JOIN cross_location ON markers.cross_location = cross_location.id AND markers.accident_year = cross_location.year AND markers.provider_code = cross_location.provider_code + LEFT JOIN cross_direction ON markers.cross_direction = cross_direction.id AND markers.accident_year = cross_direction.year AND markers.provider_code = cross_direction.provider_code + LEFT JOIN geo_area ON markers.geo_area = geo_area.id AND markers.accident_year = geo_area.year AND markers.provider_code = geo_area.provider_code + LEFT JOIN day_night ON markers.day_night = day_night.id AND markers.accident_year = day_night.year AND markers.provider_code = day_night.provider_code + LEFT JOIN day_in_week ON markers.day_in_week = day_in_week.id AND markers.accident_year = day_in_week.year AND markers.provider_code = day_in_week.provider_code + LEFT JOIN traffic_light ON markers.traffic_light = traffic_light.id AND markers.accident_year = traffic_light.year AND markers.provider_code = traffic_light.provider_code + LEFT JOIN region ON markers.region = region.id AND markers.accident_year = region.year AND markers.provider_code = region.provider_code + LEFT JOIN district ON markers.district = district.id AND markers.accident_year = district.year AND markers.provider_code = district.provider_code + LEFT JOIN natural_area ON markers.natural_area = natural_area.id AND markers.accident_year = natural_area.year AND markers.provider_code = natural_area.provider_code + LEFT JOIN municipal_status ON markers.municipal_status = municipal_status.id AND markers.accident_year = municipal_status.year AND markers.provider_code = municipal_status.provider_code + LEFT JOIN yishuv_shape ON markers.yishuv_shape = yishuv_shape.id AND markers.accident_year = yishuv_shape.year AND markers.provider_code = yishuv_shape.provider_code + LEFT JOIN accident_hour_raw ON markers.accident_hour_raw = accident_hour_raw.id AND markers.accident_year = accident_hour_raw.year AND markers.provider_code = accident_hour_raw.provider_code + LEFT JOIN provider_code ON markers.provider_code = provider_code.id;""" + + +INVOLVED_HEBREW_MARKERS_HEBREW_VIEW = """SELECT + involved_hebrew.accident_id, + involved_hebrew.provider_and_id, + involved_hebrew.provider_code, + involved_hebrew.file_type_police, + involved_hebrew.involved_type, + involved_hebrew.involved_type_hebrew, + involved_hebrew.license_acquiring_date, + involved_hebrew.age_group, + involved_hebrew.age_group_hebrew, + involved_hebrew.sex, + involved_hebrew.sex_hebrew, + involved_hebrew.vehicle_type as involve_vehicle_type, + involved_hebrew.vehicle_type_hebrew as involve_vehicle_type_hebrew, + involved_hebrew.safety_measures, + involved_hebrew.safety_measures_hebrew, + involved_hebrew.involve_yishuv_symbol, + involved_hebrew.involve_yishuv_name, + involved_hebrew.injury_severity, + involved_hebrew.injury_severity_hebrew, + involved_hebrew.injured_type, + involved_hebrew.injured_type_hebrew, + involved_hebrew.injured_position, + involved_hebrew.injured_position_hebrew, + involved_hebrew.population_type, + involved_hebrew.population_type_hebrew, + involved_hebrew.home_region as involve_home_region, + involved_hebrew.home_region_hebrew as involve_home_region_hebrew, + involved_hebrew.home_district as involve_home_district, + involved_hebrew.home_district_hebrew as involve_home_district_hebrew, + involved_hebrew.home_natural_area as involve_home_natural_area, + involved_hebrew.home_natural_area_hebrew as involve_home_natural_area_hebrew, + involved_hebrew.home_municipal_status as involve_home_municipal_status, + involved_hebrew.home_municipal_status_hebrew as involve_home_municipal_status_hebrew, + involved_hebrew.home_yishuv_shape as involve_home_yishuv_shape, + involved_hebrew.home_yishuv_shape_hebrew as involve_home_yishuv_shape_hebrew, + involved_hebrew.hospital_time, + involved_hebrew.hospital_time_hebrew, + involved_hebrew.medical_type, + involved_hebrew.medical_type_hebrew, + involved_hebrew.release_dest, + involved_hebrew.release_dest_hebrew, + involved_hebrew.safety_measures_use, + involved_hebrew.safety_measures_use_hebrew, + involved_hebrew.late_deceased, + involved_hebrew.late_deceased_hebrew, + involved_hebrew.car_id, + involved_hebrew.involve_id, + involved_hebrew.accident_year, + involved_hebrew.accident_month, + markers_hebrew.provider_code_hebrew, + markers_hebrew.accident_timestamp, + markers_hebrew.accident_type, + markers_hebrew.accident_type_hebrew, + markers_hebrew.accident_severity, + markers_hebrew.accident_severity_hebrew, + markers_hebrew.location_accuracy, + markers_hebrew.location_accuracy_hebrew, + markers_hebrew.road_type, + markers_hebrew.road_type_hebrew, + markers_hebrew.road_shape, + markers_hebrew.road_shape_hebrew, + markers_hebrew.day_type, + markers_hebrew.day_type_hebrew, + markers_hebrew.police_unit, + markers_hebrew.police_unit_hebrew, + markers_hebrew.one_lane, + markers_hebrew.one_lane_hebrew, + markers_hebrew.multi_lane, + markers_hebrew.multi_lane_hebrew, + markers_hebrew.speed_limit, + markers_hebrew.speed_limit_hebrew, + markers_hebrew.road_intactness, + markers_hebrew.road_intactness_hebrew, + markers_hebrew.road_width, + markers_hebrew.road_width_hebrew, + markers_hebrew.road_sign, + markers_hebrew.road_sign_hebrew, + markers_hebrew.road_light, + markers_hebrew.road_light_hebrew, + markers_hebrew.road_control, + markers_hebrew.road_control_hebrew, + markers_hebrew.weather, + markers_hebrew.weather_hebrew, + markers_hebrew.road_surface, + markers_hebrew.road_surface_hebrew, + markers_hebrew.road_object, + markers_hebrew.road_object_hebrew, + markers_hebrew.object_distance, + markers_hebrew.object_distance_hebrew, + markers_hebrew.didnt_cross, + markers_hebrew.didnt_cross_hebrew, + markers_hebrew.cross_mode, + markers_hebrew.cross_mode_hebrew, + markers_hebrew.cross_location, + markers_hebrew.cross_location_hebrew, + markers_hebrew.cross_direction, + markers_hebrew.cross_direction_hebrew, + markers_hebrew.road1, + markers_hebrew.road2, + markers_hebrew.km, + markers_hebrew.km_raw, + markers_hebrew.km_accurate, + markers_hebrew.road_segment_id, + markers_hebrew.road_segment_number, + markers_hebrew.road_segment_name, + markers_hebrew.road_segment_from_km, + markers_hebrew.road_segment_to_km, + markers_hebrew.road_segment_length_km, + markers_hebrew.yishuv_symbol as accident_yishuv_symbol, + markers_hebrew.yishuv_name as accident_yishuv_name, + markers_hebrew.geo_area, + markers_hebrew.geo_area_hebrew, + markers_hebrew.day_night, + markers_hebrew.day_night_hebrew, + markers_hebrew.day_in_week, + markers_hebrew.day_in_week_hebrew, + markers_hebrew.traffic_light, + markers_hebrew.traffic_light_hebrew, + markers_hebrew.region as accident_region, + markers_hebrew.region_hebrew as accident_region_hebrew, + markers_hebrew.district as accident_district, + markers_hebrew.district_hebrew as accident_district_hebrew, + markers_hebrew.natural_area as accident_natural_area, + markers_hebrew.natural_area_hebrew as accident_natural_area_hebrew, + markers_hebrew.municipal_status as accident_municipal_status, + markers_hebrew.municipal_status_hebrew as accident_municipal_status_hebrew, + markers_hebrew.yishuv_shape as accident_yishuv_shape, + markers_hebrew.yishuv_shape_hebrew as accident_yishuv_shape_hebrew, + markers_hebrew.street1, + markers_hebrew.street1_hebrew, + markers_hebrew.street2, + markers_hebrew.street2_hebrew, + markers_hebrew.non_urban_intersection, + markers_hebrew.non_urban_intersection_hebrew, + markers_hebrew.non_urban_intersection_by_junction_number, + markers_hebrew.accident_day, + markers_hebrew.accident_hour_raw, + markers_hebrew.accident_hour_raw_hebrew, + markers_hebrew.accident_hour, + markers_hebrew.accident_minute, + markers_hebrew.geom, + markers_hebrew.longitude, + markers_hebrew.latitude, + markers_hebrew.x, + markers_hebrew.y, + vehicles_hebrew.engine_volume, + vehicles_hebrew.engine_volume_hebrew, + vehicles_hebrew.manufacturing_year, + vehicles_hebrew.driving_directions, + vehicles_hebrew.driving_directions_hebrew, + vehicles_hebrew.vehicle_status, + vehicles_hebrew.vehicle_status_hebrew, + vehicles_hebrew.vehicle_attribution, + vehicles_hebrew.vehicle_attribution_hebrew, + vehicles_hebrew.seats, + vehicles_hebrew.total_weight, + vehicles_hebrew.total_weight_hebrew, + vehicles_hebrew.vehicle_type as vehicle_vehicle_type, + vehicles_hebrew.vehicle_type_hebrew as vehicle_vehicle_type_hebrew, + vehicles_hebrew.vehicle_damage, + vehicles_hebrew.vehicle_damage_hebrew + FROM involved_hebrew + LEFT JOIN markers_hebrew ON involved_hebrew.provider_code = markers_hebrew.provider_code + AND involved_hebrew.accident_id = markers_hebrew.id + AND involved_hebrew.accident_year = markers_hebrew.accident_year + LEFT JOIN vehicles_hebrew ON involved_hebrew.provider_code = vehicles_hebrew.provider_code + AND involved_hebrew.accident_id = vehicles_hebrew.accident_id + AND involved_hebrew.accident_year = vehicles_hebrew.accident_year + AND involved_hebrew.car_id = vehicles_hebrew.car_id ;""" + + +VEHICLES_MARKERS_HEBREW_VIEW = """ SELECT + markers_hebrew.accident_timestamp, + markers_hebrew.accident_type, + markers_hebrew.accident_type_hebrew, + markers_hebrew.accident_severity, + markers_hebrew.accident_severity_hebrew, + markers_hebrew.location_accuracy, + markers_hebrew.location_accuracy_hebrew, + markers_hebrew.road_type, + markers_hebrew.road_type_hebrew, + markers_hebrew.road_shape, + markers_hebrew.road_shape_hebrew, + markers_hebrew.day_type, + markers_hebrew.day_type_hebrew, + markers_hebrew.police_unit, + markers_hebrew.police_unit_hebrew, + markers_hebrew.one_lane, + markers_hebrew.one_lane_hebrew, + markers_hebrew.multi_lane, + markers_hebrew.multi_lane_hebrew, + markers_hebrew.speed_limit, + markers_hebrew.speed_limit_hebrew, + markers_hebrew.road_intactness, + markers_hebrew.road_intactness_hebrew, + markers_hebrew.road_width, + markers_hebrew.road_width_hebrew, + markers_hebrew.road_sign, + markers_hebrew.road_sign_hebrew, + markers_hebrew.road_light, + markers_hebrew.road_light_hebrew, + markers_hebrew.road_control, + markers_hebrew.road_control_hebrew, + markers_hebrew.weather, + markers_hebrew.weather_hebrew, + markers_hebrew.road_surface, + markers_hebrew.road_surface_hebrew, + markers_hebrew.road_object, + markers_hebrew.road_object_hebrew, + markers_hebrew.object_distance, + markers_hebrew.object_distance_hebrew, + markers_hebrew.didnt_cross, + markers_hebrew.didnt_cross_hebrew, + markers_hebrew.cross_mode, + markers_hebrew.cross_mode_hebrew, + markers_hebrew.cross_location, + markers_hebrew.cross_location_hebrew, + markers_hebrew.cross_direction, + markers_hebrew.cross_direction_hebrew, + markers_hebrew.road1, + markers_hebrew.road2, + markers_hebrew.km, + markers_hebrew.km_raw, + markers_hebrew.km_accurate, + markers_hebrew.road_segment_id, + markers_hebrew.road_segment_number, + markers_hebrew.road_segment_name, + markers_hebrew.road_segment_from_km, + markers_hebrew.road_segment_to_km, + markers_hebrew.road_segment_length_km, + markers_hebrew.yishuv_symbol as accident_yishuv_symbol, + markers_hebrew.yishuv_name as accident_yishuv_name, + markers_hebrew.geo_area, + markers_hebrew.geo_area_hebrew, + markers_hebrew.day_night, + markers_hebrew.day_night_hebrew, + markers_hebrew.day_in_week, + markers_hebrew.day_in_week_hebrew, + markers_hebrew.traffic_light, + markers_hebrew.traffic_light_hebrew, + markers_hebrew.region as accident_region, + markers_hebrew.region_hebrew as accident_region_hebrew, + markers_hebrew.district as accident_district, + markers_hebrew.district_hebrew as accident_district_hebrew, + markers_hebrew.natural_area as accident_natural_area, + markers_hebrew.natural_area_hebrew as accident_natural_area_hebrew, + markers_hebrew.municipal_status as accident_municipal_status, + markers_hebrew.municipal_status_hebrew as accident_municipal_status_hebrew, + markers_hebrew.yishuv_shape as accident_yishuv_shape, + markers_hebrew.yishuv_shape_hebrew as accident_yishuv_shape_hebrew, + markers_hebrew.street1, + markers_hebrew.street1_hebrew, + markers_hebrew.street2, + markers_hebrew.street2_hebrew, + markers_hebrew.non_urban_intersection, + markers_hebrew.non_urban_intersection_hebrew, + markers_hebrew.non_urban_intersection_by_junction_number, + markers_hebrew.accident_day, + markers_hebrew.accident_hour_raw, + markers_hebrew.accident_hour_raw_hebrew, + markers_hebrew.accident_hour, + markers_hebrew.accident_minute, + markers_hebrew.accident_year, + markers_hebrew.accident_month, + markers_hebrew.geom, + markers_hebrew.longitude, + markers_hebrew.latitude, + markers_hebrew.x, + markers_hebrew.y, + vehicles_hebrew.id, + vehicles_hebrew.accident_id, + vehicles_hebrew.provider_and_id, + vehicles_hebrew.provider_code, + vehicles_hebrew.file_type_police, + vehicles_hebrew.engine_volume, + vehicles_hebrew.engine_volume_hebrew, + vehicles_hebrew.manufacturing_year, + vehicles_hebrew.driving_directions, + vehicles_hebrew.driving_directions_hebrew, + vehicles_hebrew.vehicle_status, + vehicles_hebrew.vehicle_status_hebrew, + vehicles_hebrew.vehicle_attribution, + vehicles_hebrew.vehicle_attribution_hebrew, + vehicles_hebrew.seats, + vehicles_hebrew.total_weight, + vehicles_hebrew.total_weight_hebrew, + vehicles_hebrew.vehicle_type, + vehicles_hebrew.vehicle_type_hebrew, + vehicles_hebrew.vehicle_damage, + vehicles_hebrew.vehicle_damage_hebrew, + vehicles_hebrew.car_id + FROM vehicles_hebrew + INNER JOIN markers_hebrew ON vehicles_hebrew.provider_code = markers_hebrew.provider_code + AND vehicles_hebrew.accident_id = markers_hebrew.id + AND vehicles_hebrew.accident_year = markers_hebrew.accident_year ;""" + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('accident_weathers', + sa.Column('id', sa.BigInteger(), nullable=False), + sa.Column('provider_and_id', sa.BigInteger(), nullable=True), + sa.Column('provider_code', sa.Integer(), nullable=True), + sa.Column('accident_id', sa.BigInteger(), nullable=True), + sa.Column('accident_year', sa.Integer(), nullable=True), + sa.Column('rain_rate', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['accident_id', 'provider_code', 'accident_year'], ['markers.id', 'markers.provider_code', 'markers.accident_year'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index('accident_id_idx_accident_weather', 'accident_weathers', ['accident_id'], unique=False) + op.create_index('provider_and_id_idx_accident_weather', 'accident_weathers', ['provider_and_id'], unique=False) + + # Don't see this view anywhere in the code but getting this when trying to delete the involved_markers_hebrew view + # cannot drop view involved_markers_hebrew because other objects depend on it - view road_25 depends on view involved_markers_hebrew + # probably some old leftover - removing + op.execute("DROP VIEW IF EXISTS road_25") + + # we're really changing only the markers_hebrew view but we must remove and re create views dependent on it + op.execute("DROP VIEW IF EXISTS involved_markers_hebrew") + op.execute("DROP VIEW IF EXISTS vehicles_markers_hebrew") + + # the real change - remove the old view, create the new one with the weather data + op.execute("DROP VIEW IF EXISTS markers_hebrew") + op.execute(f"CREATE OR REPLACE VIEW markers_hebrew AS {MARKERS_HEBREW_VIEW_WITH_WEATHER}") + + # Recreate the dependent views + op.execute(f"CREATE OR REPLACE VIEW vehicles_markers_hebrew AS {VEHICLES_MARKERS_HEBREW_VIEW}") + op.execute(f"CREATE OR REPLACE VIEW involved_markers_hebrew AS {INVOLVED_HEBREW_MARKERS_HEBREW_VIEW}") + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('accident_weathers') + + # we're really changing only the markers_hebrew view but we must remove and re create views dependent on it + op.execute("DROP VIEW IF EXISTS involved_markers_hebrew") + op.execute("DROP VIEW IF EXISTS vehicles_markers_hebrew") + + # the real change - remove the old view, create the new one without the weather data + op.execute("DROP VIEW IF EXISTS markers_hebrew") + op.execute(f"CREATE OR REPLACE VIEW markers_hebrew AS {MARKERS_HEBREW_VIEW_WITHOUT_WEATHER}") + + # Recreate the dependent views + op.execute(f"CREATE OR REPLACE VIEW vehicles_markers_hebrew AS {VEHICLES_MARKERS_HEBREW_VIEW}") + op.execute(f"CREATE OR REPLACE VIEW involved_markers_hebrew AS {INVOLVED_HEBREW_MARKERS_HEBREW_VIEW}") + # ### end Alembic commands ### diff --git a/anyway/db_views.py b/anyway/db_views.py index 60ee321f6..51e9c5954 100644 --- a/anyway/db_views.py +++ b/anyway/db_views.py @@ -56,6 +56,7 @@ class Views(object): markers.km, markers.km_raw, markers.km_accurate, + accident_weathers.rain_rate as accident_rain_rate, road_segments.segment_id as road_segment_id, road_segments.segment as road_segment_number, road_segments.from_name || ' - ' || road_segments.to_name as road_segment_name, @@ -121,6 +122,7 @@ class Views(object): LEFT JOIN road_light ON markers.road_light = road_light.id AND markers.accident_year = road_light.year AND markers.provider_code = road_light.provider_code LEFT JOIN road_control ON markers.road_control = road_control.id AND markers.accident_year = road_control.year AND markers.provider_code = road_control.provider_code LEFT JOIN weather ON markers.weather = weather.id AND markers.accident_year = weather.year AND markers.provider_code = weather.provider_code + LEFT JOIN accident_weathers ON accident_weathers.provider_code = markers.provider_code AND accident_weathers.accident_id = markers.id AND accident_weathers.accident_year = markers.accident_year LEFT JOIN road_surface ON markers.road_surface = road_surface.id AND markers.accident_year = road_surface.year AND markers.provider_code = road_surface.provider_code LEFT JOIN road_object ON markers.road_object = road_object.id AND markers.accident_year = road_object.year AND markers.provider_code = road_object.provider_code LEFT JOIN object_distance ON markers.object_distance = object_distance.id AND markers.accident_year = object_distance.year AND markers.provider_code = object_distance.provider_code diff --git a/anyway/models.py b/anyway/models.py index 8d10abe79..f881545ac 100755 --- a/anyway/models.py +++ b/anyway/models.py @@ -239,6 +239,7 @@ class AccidentMarker(MarkerMixin, Base): cross_direction = Column(Integer()) involved = relationship("Involved") vehicles = relationship("Vehicle") + weather_data = relationship("AccidentWeather", uselist=False) video_link = Column(Text()) road1 = Column(Integer()) road2 = Column(Integer()) @@ -649,6 +650,47 @@ def parse(cls, data): ) +class AccidentWeather(Base): + __tablename__ = "accident_weathers" + id = Column(BigInteger(), primary_key=True) + provider_and_id = Column(BigInteger()) + provider_code = Column(Integer()) + accident_id = Column(BigInteger()) + accident_year = Column(Integer()) + rain_rate = Column(Integer()) + __table_args__ = ( + ForeignKeyConstraint( + [accident_id, provider_code, accident_year], + [AccidentMarker.id, AccidentMarker.provider_code, AccidentMarker.accident_year], + ondelete="CASCADE", + ), + Index("accident_id_idx_accident_weather", "accident_id", unique=False), + Index("provider_and_id_idx_accident_weather", "provider_and_id", unique=False), + {}, + ) + + def serialize(self): + return { + "id": self.id, + "provider_code": self.provider_code, + "accident_id": self.accident_id, + "rain_rate": self.rain_rate, + } + + # Flask-Login integration + def is_authenticated(self): + return True + + def is_active(self): + return True + + def is_anonymous(self): + return False + + def get_id(self): + return self.id + + class DiscussionMarker(MarkerMixin, Base): __tablename__ = "discussions" __table_args__ = ( @@ -1772,6 +1814,7 @@ class AccidentMarkerView(Base): road_segment_id = Column(Integer()) road_segment_name = Column(Text()) road_segment_number = Column(Integer()) + accident_rain_rate = Column(Integer()) def serialize(self): return { @@ -1866,6 +1909,7 @@ def serialize(self): "longitude": self.longitude, "x": self.x, "y": self.y, + "accident_rain_rate": self.accident_rain_rate, } diff --git a/anyway/parsers/cbs/executor.py b/anyway/parsers/cbs/executor.py index 198dfd98f..98962b93c 100644 --- a/anyway/parsers/cbs/executor.py +++ b/anyway/parsers/cbs/executor.py @@ -15,6 +15,7 @@ from sqlalchemy import or_, and_ from anyway.parsers.cbs import preprocessing_cbs_files, importmail_cbs +from anyway.parsers.cbs.weather_data import ensure_accidents_weather_data from anyway import field_names, localization from anyway.backend_constants import BE_CONST from anyway.models import ( @@ -1146,6 +1147,8 @@ def main( ) logging.info("Total: {0} items in {1}".format(total, time_delta(started))) + ensure_accidents_weather_data() + create_views() except Exception as ex: print("Exception occured while loading the cbs data: {0}".format(str(ex))) diff --git a/anyway/parsers/cbs/weather_data.py b/anyway/parsers/cbs/weather_data.py new file mode 100644 index 000000000..ad7276778 --- /dev/null +++ b/anyway/parsers/cbs/weather_data.py @@ -0,0 +1,48 @@ +import logging + +from anyway.app_and_db import db +from anyway.models import ( + AccidentMarker, + AccidentWeather, +) +from anyway.parsers.cbs.weather_interpolator import get_weather + + +def ensure_accidents_weather_data(start_date=None, filters=None): + """ + :param start_date: Add start date filter to the query that lists accident markers to add weather data to + :param filters: additional filters to add to the query that lists accident markers to add weather data to + This is used mainly for testing - format DD-MM-YYYY + :returns: int representing the number of accidents to which weather data was added + """ + logging.info(f"Ensuring accidents weather data {start_date} {filters}") + query = db.session.query(AccidentMarker).filter(AccidentMarker.weather_data == None) + if start_date: + query = query.filter(AccidentMarker.created > start_date) + if filters is not None: + query = query.filter(*filters) + accident_markers_to_update = query.all() + if accident_markers_to_update: + logging.debug( + f"Found accident markers without weather data. {len(accident_markers_to_update)}" + ) + accidents_weather_data = [] + for accident_marker in query.all(): + weather_data = get_weather( + accident_marker.latitude, accident_marker.longitude, accident_marker.created.isoformat() + ) + accidents_weather_data.append( + { + "accident_id": accident_marker.id, + "provider_and_id": accident_marker.provider_and_id, + "provider_code": accident_marker.provider_code, + "accident_year": accident_marker.accident_year, + "rain_rate": weather_data["rain"], + } + ) + if accidents_weather_data: + logging.debug(f"Adding weather data to accidents. {accidents_weather_data}") + db.session.bulk_insert_mappings(AccidentWeather, accidents_weather_data) + db.session.commit() + logging.debug("Finished filling accidents weather data") + return len(accident_markers_to_update) if accident_markers_to_update else 0 diff --git a/main.py b/main.py index 1f8c52a74..485d1a5ae 100755 --- a/main.py +++ b/main.py @@ -269,6 +269,20 @@ def waze_data(from_s3, start_date, end_date): return ingest_waze_from_api() +@process.command() +@click.option( + "--start_date", default=None, type=valid_date, help="The Start Date - format DD-MM-YYYY" +) +def weather_data(start_date): + """ + Looping on the accidents from the cbs and ensuring they have weather data + Start date can be given to filter out accident before the given date + """ + + from anyway.parsers.cbs.weather_data import ensure_accidents_weather_data + ensure_accidents_weather_data(start_date) + + @process.command() @click.argument("filename", type=str, default="static/data/embedded_reports/embedded_reports.csv") def embedded_reports(filename): diff --git a/tests/test_weather_data.py b/tests/test_weather_data.py new file mode 100644 index 000000000..1a329a354 --- /dev/null +++ b/tests/test_weather_data.py @@ -0,0 +1,89 @@ +import logging +from datetime import datetime + +from sqlalchemy import func + +from anyway.app_and_db import db +from anyway.backend_constants import BE_CONST +from anyway.models import ( + AccidentMarker, + AccidentWeather, +) +from anyway.parsers.cbs.weather_data import ensure_accidents_weather_data + + +class TestWeatherData: + + def _insert_accident_marker(self, created=None): + logging.info("Inserting test accident marker") + + # to not conflict with existing ids find max value and add one + accident_marker_id = db.session.query(func.max(AccidentMarker.id)).one()[0] + 1 + logging.debug(f"Calculated id for accident marker: {accident_marker_id}") + accident_marker = AccidentMarker( + id=accident_marker_id, + provider_and_id=0, + provider_code=BE_CONST.CBS_ACCIDENT_TYPE_1_CODE, + accident_year=2020, + latitude=32.0580, + longitude=34.7588, + ) + if created: + accident_marker.created = created + + db.session.add(accident_marker) + db.session.commit() + + return accident_marker_id + + def test_ensure_accidents_weather_data(self): + accident_marker_id = self._insert_accident_marker() + + logging.debug("Verifying accident marker does not have weather data") + accident_marker = db.session.query(AccidentMarker).filter(AccidentMarker.id == accident_marker_id).one() + assert accident_marker.weather_data is None + + # the test DB may have other markers we don't want to add weather data to, so ensuring only our marker + filters = ( + AccidentMarker.id == accident_marker_id, + ) + number_of_accidents_updated = ensure_accidents_weather_data(filters=filters) + assert number_of_accidents_updated == 1 + + logging.debug("Verifying weather data added to accident marker") + accident_marker = db.session.query(AccidentMarker).filter(AccidentMarker.id == accident_marker_id).one() + assert accident_marker.weather_data is not None + + logging.debug(f"Weather data verified {accident_marker.weather_data}") + + logging.debug("Verifying another run of ensure weather data changes nothing") + number_of_accidents_updated = ensure_accidents_weather_data(filters=filters) + assert number_of_accidents_updated == 0 + + logging.debug("Removing test data") + db.session.query(AccidentMarker).filter(AccidentMarker.id == accident_marker_id).delete() + db.session.query(AccidentWeather).filter(AccidentWeather.id == accident_marker.weather_data.id).delete() + db.session.commit() + + def test_ensure_accidents_weather_data_with_date_filter(self): + old_date = datetime(year=2010, day=1, month=1) + new_date = datetime(year=2020, day=1, month=1) + old_accident_marker_id = self._insert_accident_marker(old_date) + new_accident_marker_id = self._insert_accident_marker(new_date) + + # the test DB may have other markers we don't want to add weather data to, so ensuring only our marker + filters = ( + AccidentMarker.id.in_([old_accident_marker_id, new_accident_marker_id]), + ) + filter_date = datetime(year=2015, day=1, month=1) + number_of_accidents_updated = ensure_accidents_weather_data(filter_date, filters) + + # only one accident will be updated since only one is after the filter date + assert number_of_accidents_updated == 1 + + logging.debug("Removing test data") + accident_marker = db.session.query(AccidentMarker).filter(AccidentMarker.id == new_accident_marker_id).one() + db.session.query(AccidentWeather).filter(AccidentWeather.id == accident_marker.weather_data.id).delete() + db.session.query(AccidentMarker).filter(AccidentMarker.id == new_accident_marker_id).delete() + db.session.query(AccidentMarker).filter(AccidentMarker.id == old_accident_marker_id).delete() + db.session.commit() From 2efdb943ffcf3f5b97cf6e56c21209d36b50ec44 Mon Sep 17 00:00:00 2001 From: hedingber Date: Wed, 18 Nov 2020 21:28:02 +0200 Subject: [PATCH 3/3] Add infographic --- anyway/infographics_utils.py | 82 ++++++++++++++++++++++++++++++++---- 1 file changed, 73 insertions(+), 9 deletions(-) diff --git a/anyway/infographics_utils.py b/anyway/infographics_utils.py index 50a071248..9c0bff538 100755 --- a/anyway/infographics_utils.py +++ b/anyway/infographics_utils.py @@ -49,6 +49,7 @@ class WidgetId(Enum): vision_zero = auto() accident_count_by_driver_type = auto() accident_count_by_car_type = auto() + rain_accidents_by_severity = auto() injured_accidents_with_pedestrians = auto() accident_severity_by_cross_location = auto() motorcycle_accidents_vs_all_accidents = auto() @@ -735,8 +736,10 @@ def __init__(self, request_params: RequestParams): } def generate_items(self) -> None: - self.items = AccidentCountByCarTypeWidget.get_stats_accidents_by_car_type_with_national_data( - self.request_params + self.items = ( + AccidentCountByCarTypeWidget.get_stats_accidents_by_car_type_with_national_data( + self.request_params + ) ) @staticmethod @@ -759,8 +762,10 @@ def get_stats_accidents_by_car_type_with_national_data( data_by_segment = AccidentCountByCarTypeWidget.percentage_accidents_by_car_type( involved_by_vehicle_type_data ) - national_data = AccidentCountByCarTypeWidget.percentage_accidents_by_car_type_national_data_cache( - start_time, end_time + national_data = ( + AccidentCountByCarTypeWidget.percentage_accidents_by_car_type_national_data_cache( + start_time, end_time + ) ) for k, v in national_data.items(): # pylint: disable=W0612 @@ -1041,6 +1046,66 @@ def pedestrian_injured_in_junctions_mock_data(): # Temporary for Frontend ] +@WidgetCollection.register +class AccidentCausedByRainWidget(Widget): + # the rain rate threshold after which we count the accident as a cause of the rain + ACCIDENT_RAIN_RATE_THRESHOLD = 4 + + def __init__(self, request_params: RequestParams): + super().__init__(request_params, WidgetId.rain_accidents_by_severity) + self.rank = 24 + self.text = { + "title": "תאונות שהתרחשו בזמן גשם במקטע " + + self.request_params.location_info["road_segment_name"] + } + + def generate_items(self) -> None: + self.items = AccidentCausedByRainWidget.stats_accidents_caused_by_rain_by_severity( + self.request_params.location_info, + self.request_params.start_time, + self.request_params.end_time, + ) + + @staticmethod + def stats_accidents_caused_by_rain_by_severity(location_info, start_time, end_time): + all_segment_accidents = get_accidents_stats( + table_obj=AccidentMarkerView, + filters=location_info, + start_time=start_time, + end_time=end_time, + raw=True, + ) + + severity_to_severity_hebrew = {} + accidents_by_severity = defaultdict(int) + rain_accidents_by_severity = defaultdict(int) + for accident in all_segment_accidents: + severity = accident["accident_severity"] + severity_hebrew = accident["accident_severity_hebrew"] + severity_to_severity_hebrew[severity] = severity_hebrew + accidents_by_severity[severity] += 1 + if ( + accident["accident_rain_rate"] + > AccidentCausedByRainWidget.ACCIDENT_RAIN_RATE_THRESHOLD + ): + rain_accidents_by_severity[severity] += 1 + + stats = [] + for severity, rain_accidents_amount in rain_accidents_by_severity.items(): + stats.append( + { + "severity": severity, + "severity_hebrew": severity_to_severity_hebrew[severity], + "amount_of_accidents_caused_by_rain": rain_accidents_amount, + "accidents_caused_by_rain_percentage": int( + rain_accidents_amount / accidents_by_severity[severity] * 100 + ), + } + ) + + return stats + + def extract_news_flash_location(news_flash_obj): resolution = news_flash_obj.resolution or None if not news_flash_obj or not resolution or resolution not in resolution_dict: @@ -1074,7 +1139,7 @@ def get_query(table_obj, filters, start_time, end_time): def get_accidents_stats( - table_obj, filters=None, group_by=None, count=None, start_time=None, end_time=None + table_obj, filters=None, group_by=None, count=None, start_time=None, end_time=None, raw=False ): filters = filters or {} filters["provider_code"] = [ @@ -1088,10 +1153,9 @@ def get_accidents_stats( query = query.with_entities(group_by, func.count(count)) df = pd.read_sql_query(query.statement, query.session.bind) df.rename(columns={"count_1": "count"}, inplace=True) # pylint: disable=no-member - df.columns = [c.replace("_hebrew", "") for c in df.columns] - return ( # pylint: disable=no-member - df.to_dict(orient="records") if group_by or count else df.to_dict() - ) + if not raw: + df.columns = [c.replace("_hebrew", "") for c in df.columns] + return df.to_dict(orient="records") # pylint: disable=no-member def get_injured_filters(location_info):