From 2d810751513d5fc156c3cca9e56a3b15d4f46d5b Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 2 Oct 2024 13:08:44 -0700 Subject: [PATCH 01/15] Update the STUDY_CONFIG in docker-compose.dev.yml config file with the program/study name from the database name of the dataset. --- viz_scripts/docker/load_mongodump.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/viz_scripts/docker/load_mongodump.sh b/viz_scripts/docker/load_mongodump.sh index 64fc71a..42f26b4 100644 --- a/viz_scripts/docker/load_mongodump.sh +++ b/viz_scripts/docker/load_mongodump.sh @@ -53,6 +53,9 @@ echo "Database Name: $DB_NAME" DB_HOST="mongodb://db/$DB_NAME" sed -i.bak "s|DB_HOST=.*|DB_HOST=$DB_HOST|" "$CONFIG_FILE" +STUDY_CONFIG=$(echo "$DB_NAME" | sed -E 's/openpath_prod_(.*)$/\1/' | tr '_' '-') +sed -i.bak "s|STUDY_CONFIG=.*|STUDY_CONFIG=$STUDY_CONFIG|" "$CONFIG_FILE" + echo "Updated docker-compose file:" cat "$CONFIG_FILE" From 242a7eae155a175e95b1d8534641fb9aaccdd257 Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 2 Oct 2024 13:29:18 -0700 Subject: [PATCH 02/15] Update comment for updating docker-compose config file with actual STUDY_CONFIG --- viz_scripts/docker/load_mongodump.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/viz_scripts/docker/load_mongodump.sh b/viz_scripts/docker/load_mongodump.sh index 42f26b4..dc5da1b 100644 --- a/viz_scripts/docker/load_mongodump.sh +++ b/viz_scripts/docker/load_mongodump.sh @@ -53,6 +53,7 @@ echo "Database Name: $DB_NAME" DB_HOST="mongodb://db/$DB_NAME" sed -i.bak "s|DB_HOST=.*|DB_HOST=$DB_HOST|" "$CONFIG_FILE" +# Update the docker-compose configuration file with the actual STUDY_CONFIG STUDY_CONFIG=$(echo "$DB_NAME" | sed -E 's/openpath_prod_(.*)$/\1/' | tr '_' '-') sed -i.bak "s|STUDY_CONFIG=.*|STUDY_CONFIG=$STUDY_CONFIG|" "$CONFIG_FILE" From 4c05924b781cddbe76c98535b3520b1ee1052379 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 11:37:34 -0700 Subject: [PATCH 03/15] Load read_json_resource() from e-mission-common - Pass labels instead of dynamic_labels to the notebooks - labels can either be: - Custom labels from the program deployment config, from nrel-openpath-deploy-configs - In case, we do not have custom labels for the program/study, load default labels from e-mission-common with read_json_resource --- viz_scripts/bin/generate_plots.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/viz_scripts/bin/generate_plots.py b/viz_scripts/bin/generate_plots.py index 38d66c6..65815b2 100644 --- a/viz_scripts/bin/generate_plots.py +++ b/viz_scripts/bin/generate_plots.py @@ -7,7 +7,8 @@ import json import os import sys - +import asyncio +import emcommon.util as emcu # Configuration settings to use for all generated plots by this instance # This could also be specified as a parser argument, if we want to generate plots for all programs from one instance @@ -43,7 +44,11 @@ # dynamic_labels can be referenced from # https://github.com/e-mission/nrel-openpath-deploy-configs/blob/main/label_options/example-study-label-options.json -dynamic_labels = { } +labels = { } + +async def load_default_label_options(): + labels = await emcu.read_json_resource("label-options.default.json") + return labels # Check if the dynamic config contains dynamic labels 'label_options' # Parse through the dynamic_labels_url: @@ -54,10 +59,16 @@ if req.status_code != 200: print(f"Unable to download dynamic_labels_url, status code: {req.status_code} for {STUDY_CONFIG}") else: - dynamic_labels = json.loads(req.text) + labels = json.loads(req.text) print(f"Dynamic labels download was successful for nrel-openpath-deploy-configs: {STUDY_CONFIG}" ) else: - print(f"label_options is unavailable for the dynamic_config in {STUDY_CONFIG}") + # load default labels from e-mission-common + # https://raw.githubusercontent.com/JGreenlee/e-mission-common/refs/heads/master/src/emcommon/resources/label-options.default.json + labels = asyncio.run(load_default_label_options()) + if not labels: + print(f"Unable to load labels for : {STUDY_CONFIG}") + else: + print(f"Labels loading was successful for nrel-openpath-deploy-configs: {STUDY_CONFIG}") if args.date is None: start_date = arrow.get(int(dynamic_config['intro']['start_year']), @@ -88,7 +99,7 @@ def compute_for_date(month, year): study_type=dynamic_config['intro']['program_or_study'], mode_of_interest=mode_studied, include_test_users=dynamic_config.get('metrics', {}).get('include_test_users', False), - dynamic_labels = dynamic_labels, + labels = labels, use_imperial = dynamic_config.get('display_config', {}).get('use_imperial', True), sensed_algo_prefix=dynamic_config.get('metrics', {}).get('sensed_algo_prefix', "cleaned"), bluetooth_only = dynamic_config.get('tracking', {}).get('bluetooth_only', False), From 3b5b346f28de2b59f9a81b9735ea97531699ef85 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 12:56:24 -0700 Subject: [PATCH 04/15] Remove call to read_json_resource("label-options.default.json") from e-mission-common in scaffolding.py Replace dynamic_labels with "labels" - labels could be either default labels or dynamic labels (provided for program/study) depending on availability in config file - Use labels passed from notebook instead of calling read_json_resource("label-options.default.json") from e-mission-common Remove async and await for mapping_color_labels() and translate_values_to_labels() Remove filtering for dynamic_labels availability, as we will only have either labels or dynamic_labels. Update load_viz_notebook_* functions to use labels instead of dynamic_labels --- viz_scripts/scaffolding.py | 56 ++++++++++++-------------------------- 1 file changed, 18 insertions(+), 38 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 3e46eba..7d19424 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -54,9 +54,8 @@ def get_participant_uuids(program, load_test_users): disp.display(participant_list.user_email) return participant_uuid_str -async def add_base_mode_footprint(trip_list): +async def add_base_mode_footprint(trip_list, labels): #TODO filter ahead of this so only labeled trips get a footprint OR display uncertainties - labels = await emcu.read_json_resource("label-options.default.json") value_to_basemode = {mode["value"]: mode.get("base_mode", mode.get("baseMode", "UNKNOWN")) for mode in labels["MODE"]} for trip in trip_list: @@ -84,11 +83,11 @@ async def add_base_mode_footprint(trip_list): return trip_list -async def load_all_confirmed_trips(tq, add_footprint): +async def load_all_confirmed_trips(tq, labels, add_footprint): agg = esta.TimeSeries.get_aggregate_time_series() result_it = agg.find_entries(["analysis/confirmed_trip"], tq) if add_footprint: - processed_list = await add_base_mode_footprint(list(result_it)) + processed_list = await add_base_mode_footprint(list(result_it), labels) all_ct = agg.to_data_df("analysis/confirmed_trip", processed_list) else: all_ct = agg.to_data_df("analysis/confirmed_trip", result_it) @@ -96,9 +95,9 @@ async def load_all_confirmed_trips(tq, add_footprint): disp.display(all_ct.head()) return all_ct -async def load_all_participant_trips(program, tq, load_test_users, add_footprint=False): +async def load_all_participant_trips(program, tq, load_test_users, labels, add_footprint=False): participant_list = get_participant_uuids(program, load_test_users) - all_ct = await load_all_confirmed_trips(tq, add_footprint) + all_ct = await load_all_confirmed_trips(tq, labels, add_footprint) # CASE 1 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if len(all_ct) == 0: return all_ct @@ -177,7 +176,7 @@ def _select_max_label(row): unique_users = lambda df: len(df.user_id.unique()) if "user_id" in df.columns else 0 trip_label_count = lambda s, df: len(df[s].dropna()) if s in df.columns else 0 -async def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, include_test_users=False, add_footprint=False): +async def load_viz_notebook_data(year, month, program, study_type, labels, include_test_users=False, add_footprint=False): #TODO - see how slow the loading the footprint is compared to just the baseMode, and evaluate if passing param around is needed """ Inputs: year/month/program/study_type = parameters from the visualization notebook @@ -187,11 +186,11 @@ async def load_viz_notebook_data(year, month, program, study_type, dynamic_label """ # Access database tq = get_time_query(year, month) - participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, add_footprint) + participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, labels, add_footprint) labeled_ct = filter_labeled_trips(participant_ct_df) expanded_ct = expand_userinputs(labeled_ct) expanded_ct = data_quality_check(expanded_ct) - expanded_ct = await map_trip_data(expanded_ct, study_type, dynamic_labels) + expanded_ct = map_trip_data(expanded_ct, study_type, labels) # Document data quality file_suffix = get_file_suffix(year, month, program) @@ -211,19 +210,12 @@ async def load_viz_notebook_data(year, month, program, study_type, dynamic_label return expanded_ct, file_suffix, quality_text, debug_df -async def map_trip_data(expanded_trip_df, study_type, dynamic_labels): +def map_trip_data(expanded_trip_df, study_type, labels): # Change meters to miles # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if "distance" in expanded_trip_df.columns: unit_conversions(expanded_trip_df) - # Select the labels from dynamic_labels is available, - # else get it from emcommon/resources/label-options.default.json - if (len(dynamic_labels)): - labels = dynamic_labels - else: - labels = await emcu.read_json_resource("label-options.default.json") - # Map new mode labels with translations dictionary from dynamic_labels # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if "mode_confirm" in expanded_trip_df.columns: @@ -254,7 +246,7 @@ async def map_trip_data(expanded_trip_df, study_type, dynamic_labels): return expanded_trip_df -async def load_viz_notebook_inferred_data(year, month, program, study_type, dynamic_labels, include_test_users=False): +async def load_viz_notebook_inferred_data(year, month, program, study_type, labels, include_test_users=False): """ Inputs: year/month/program/study_type = parameters from the visualization notebook dic_* = label mappings; if dic_pur is included it will be used to recode trip purpose @@ -263,10 +255,10 @@ async def load_viz_notebook_inferred_data(year, month, program, study_type, dyna """ # Access database tq = get_time_query(year, month) - participant_ct_df = await load_all_participant_trips(program, tq, include_test_users) + participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, labels) inferred_ct = filter_inferred_trips(participant_ct_df) expanded_it = expand_inferredlabels(inferred_ct) - expanded_it = await map_trip_data(expanded_it, study_type, dynamic_labels) + expanded_it = map_trip_data(expanded_it, study_type, labels) # Document data quality file_suffix = get_file_suffix(year, month, program) @@ -307,15 +299,9 @@ def translate_labels(labels): # Function: Maps "MODE", "PURPOSE", and "REPLACED_MODE" to colors. # Input: dynamic_labels # Output: Dictionary mapping between color with mode/purpose/sensed -async def mapping_color_labels(dynamic_labels = {}, unique_keys = []): - # Load default options from e-mission-common - labels = await emcu.read_json_resource("label-options.default.json") +def mapping_color_labels(labels = {}, unique_keys = []): sensed_values = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "OTHER", "INVALID"] - # If dynamic_labels are provided, then we will use the dynamic labels for mapping - if len(dynamic_labels) > 0: - labels = dynamic_labels - # Load base mode values and purpose values mode_values = [mode["value"] for mode in labels["MODE"]] if "MODE" in labels else [] purpose_values = [mode["value"] for mode in labels["PURPOSE"]] if "PURPOSE" in labels else [] @@ -343,13 +329,7 @@ async def mapping_color_labels(dynamic_labels = {}, unique_keys = []): ], adjustment_range=[1,1.8]) return colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble -async def translate_values_to_labels(dynamic_labels, language="en"): - # Load default options from e-mission-common - labels = await emcu.read_json_resource("label-options.default.json") - - # If dynamic_labels are provided, then we will use the dynamic labels for mapping - if len(dynamic_labels) > 0: - labels = dynamic_labels +def translate_values_to_labels(labels, language="en"): # Mapping between values and translations for display on plots (for Mode) values_to_translations_mode = mapping_labels(labels, "MODE") # Mapping between values and translations for display on plots (for Purpose) @@ -373,14 +353,14 @@ def mapping_color_surveys(dic_options): return colors -async def load_viz_notebook_sensor_inference_data(year, month, program, include_test_users=False, sensed_algo_prefix="cleaned"): +async def load_viz_notebook_sensor_inference_data(year, month, program, labels, include_test_users=False, sensed_algo_prefix="cleaned"): """ Inputs: year/month/program = parameters from the visualization notebook Pipeline to load and process the data before use in sensor-based visualization notebooks. """ tq = get_time_query(year, month) - participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, False) + participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, labels, False) expanded_ct = participant_ct_df print(f"Loaded expanded_ct with length {len(expanded_ct)} for {tq}") @@ -427,14 +407,14 @@ async def load_viz_notebook_sensor_inference_data(year, month, program, include_ return expanded_ct, file_suffix, quality_text, debug_df -async def load_viz_notebook_survey_data(year, month, program, include_test_users=False): +async def load_viz_notebook_survey_data(year, month, program, labels, include_test_users=False): """ Inputs: year/month/program/test users = parameters from the visualization notebook Returns: df of all trips taken by participants, df of all trips with user_input """ tq = get_time_query(year, month) - participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, False) + participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, labels, False) labeled_ct = filter_labeled_trips(participant_ct_df) # Document data quality From 413420d41382d0308a9d2273130c8ad6af1bfc6b Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 13:00:27 -0700 Subject: [PATCH 05/15] Replace dynamic_labels with labels. Remove await from mapping_color_labels() and translate_values_to_labels(). --- viz_scripts/generic_metrics.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index b0992f1..370d6d7 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -30,7 +30,7 @@ "program = \"default\"\n", "study_type = \"study\"\n", "include_test_users = False\n", - "dynamic_labels = {}\n", + "labels = {}\n", "use_imperial = True\n", "sensed_algo_prefix = \"cleaned\"\n", "survey_info = {}" @@ -97,8 +97,8 @@ "metadata": {}, "outputs": [], "source": [ - "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(dynamic_labels)\n", - "values_to_translations, value_to_translations_purpose, values_to_translations_replaced = await scaffolding.translate_values_to_labels(dynamic_labels)" + "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = scaffolding.mapping_color_labels(labels)\n", + "values_to_translations, value_to_translations_purpose, values_to_translations_replaced = scaffolding.translate_values_to_labels(labels)" ] }, { @@ -120,7 +120,7 @@ " month,\n", " program,\n", " study_type,\n", - " dynamic_labels,\n", + " labels,\n", " include_test_users=include_test_users)" ] }, @@ -165,7 +165,7 @@ " month,\n", " program,\n", " study_type,\n", - " dynamic_labels,\n", + " labels,\n", " include_test_users=include_test_users)" ] }, From db6797795b932d7f8202fa06d929ec30847c4d9d Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 13:02:37 -0700 Subject: [PATCH 06/15] Replace dynamic_labels with labels in energy_calculations notebook. --- viz_scripts/energy_calculations.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index 79eccf1..569e1c5 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -32,7 +32,7 @@ "mode_of_interest = \"e-bike\"\n", "include_test_users = False\n", "is_debug_mode = False\n", - "dynamic_labels = {}\n", + "labels = {}\n", "use_imperial = True" ] }, @@ -91,7 +91,7 @@ " month,\n", " program,\n", " study_type,\n", - " dynamic_labels,\n", + " labels,\n", " include_test_users=include_test_users,\n", " add_footprint=True)\n", "\n", @@ -224,9 +224,9 @@ " if (is_debug_mode == True):\n", " ebco2_debug = ebco2_compute(data_eb, weight_unit_debug)\n", " if (weight_unit == 'kg'):\n", - " scaffolding.print_CO2_emission_calculations(data_eb, ebco2_debug, ebco2, dynamic_labels)\n", + " scaffolding.print_CO2_emission_calculations(data_eb, ebco2_debug, ebco2, labels)\n", " else:\n", - " scaffolding.print_CO2_emission_calculations(data_eb, ebco2, ebco2_debug, dynamic_labels)\n", + " scaffolding.print_CO2_emission_calculations(data_eb, ebco2, ebco2_debug, labels)\n", "\n", "try:\n", " if use_imperial:\n", From ab70832f0abfe3295697fa83036255f636bf0f70 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 13:06:22 -0700 Subject: [PATCH 07/15] Replace dynamic_labels with labels in generic_timeseries notebook. Remove filter for dynamic_labels, since we pass either dynamic_labels or labels based on the availability of dynamic labels in program/study config. --- viz_scripts/generic_timeseries.ipynb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index 54ab23a..2171c9a 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -28,7 +28,7 @@ "study_type = \"study\"\n", "mode_of_interest = None\n", "include_test_users = False\n", - "dynamic_labels = {}\n", + "labels = {}\n", "use_imperial = False\n", "sensed_algo_prefix = \"cleaned\"" ] @@ -84,7 +84,7 @@ " month,\n", " program,\n", " study_type,\n", - " dynamic_labels,\n", + " labels,\n", " include_test_users=include_test_users,\n", " add_footprint=True)\n", "expanded_ct = scaffolding.unpack_energy_emissions(expanded_ct) if \"mode_confirm_footprint\" in expanded_ct.columns else expanded_ct\n", @@ -143,10 +143,6 @@ " expanded_ct_sensed.user_id = pd.Categorical(expanded_ct_sensed.user_id)\n", " expanded_ct_sensed.date_time = pd.Categorical(expanded_ct_sensed.date_time)\n", "\n", - "if len(dynamic_labels) > 0:\n", - " labels=dynamic_labels\n", - "else:\n", - " labels = await emcu.read_json_resource(\"label-options.default.json\")\n", "dic_mode_mapping = scaffolding.mapping_labels(labels, \"MODE\")\n", "\n", "if \"Mode_confirm\" in expanded_ct.columns:\n", From a050320740c23c82dd3805d1b3b2702f1e6b74c0 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 13:11:11 -0700 Subject: [PATCH 08/15] Replace dynamic_labels with labels in mode_specific_metrics notebook. Remove await for mapping_color_labels() and translate_values_to_labels(). --- viz_scripts/mode_specific_metrics.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/viz_scripts/mode_specific_metrics.ipynb b/viz_scripts/mode_specific_metrics.ipynb index a165630..0753a60 100644 --- a/viz_scripts/mode_specific_metrics.ipynb +++ b/viz_scripts/mode_specific_metrics.ipynb @@ -31,7 +31,7 @@ "study_type = \"program\"\n", "mode_of_interest = \"e-bike\"\n", "include_test_users = False\n", - "dynamic_labels = { }\n", + "labels = { }\n", "use_imperial = True" ] }, @@ -95,8 +95,8 @@ "metadata": {}, "outputs": [], "source": [ - "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(dynamic_labels)\n", - "values_to_translations, value_to_translations_purpose, value_to_translations_replaced = await scaffolding.translate_values_to_labels(dynamic_labels)" + "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = scaffolding.mapping_color_labels(labels)\n", + "values_to_translations, value_to_translations_purpose, value_to_translations_replaced = scaffolding.translate_values_to_labels(labels)" ] }, { @@ -118,7 +118,7 @@ " month,\n", " program,\n", " study_type,\n", - " dynamic_labels,\n", + " labels,\n", " include_test_users=include_test_users)" ] }, @@ -141,7 +141,7 @@ " month,\n", " program,\n", " study_type,\n", - " dynamic_labels,\n", + " labels,\n", " include_test_users=include_test_users)" ] }, From 595876dcf7a0802be9071f2fe4e81c1ca34caafa Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 13:13:17 -0700 Subject: [PATCH 09/15] Replace dynamic_labels with labels. Remove check for dynamic_labels since we pass dynamic labels or labels based on the availabilty in study/program's config file. --- viz_scripts/mode_specific_timeseries.ipynb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/viz_scripts/mode_specific_timeseries.ipynb b/viz_scripts/mode_specific_timeseries.ipynb index 2342951..69af5ea 100644 --- a/viz_scripts/mode_specific_timeseries.ipynb +++ b/viz_scripts/mode_specific_timeseries.ipynb @@ -28,7 +28,7 @@ "study_type = \"program\"\n", "mode_of_interest = \"e-bike\"\n", "include_test_users = False\n", - "dynamic_labels = { }\n", + "labels = { }\n", "use_imperial = False" ] }, @@ -94,7 +94,7 @@ " month,\n", " program,\n", " study_type,\n", - " dynamic_labels,\n", + " labels,\n", " include_test_users=include_test_users)" ] }, @@ -177,10 +177,6 @@ " mode_distance_interest = mode_distance[mode_distance['mode_confirm']==mode_of_interest].copy()\n", "\n", " # Mapping new mode labels with dictionaries\n", - " if len(dynamic_labels) > 0:\n", - " labels=dynamic_labels\n", - " else:\n", - " labels = await emcu.read_json_resource(\"label-options.default.json\")\n", " dic_mode_mapping = scaffolding.mapping_labels(labels, \"MODE\")\n", " mode_counts['Mode_confirm'] = mode_counts['mode_confirm'].map(dic_mode_mapping)\n", " mode_counts_interest['Mode_confirm'] = mode_counts_interest['mode_confirm'].map(dic_mode_mapping)\n", From 876bdc2b3d8e9b8d75c64a2d6156c33679b0ebec Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 13:15:13 -0700 Subject: [PATCH 10/15] Replace dynamic_labels with labels. Remove await from mapping_color_labels(). --- viz_scripts/survey_metrics.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/viz_scripts/survey_metrics.ipynb b/viz_scripts/survey_metrics.ipynb index cf724a0..2ff91b6 100644 --- a/viz_scripts/survey_metrics.ipynb +++ b/viz_scripts/survey_metrics.ipynb @@ -30,7 +30,7 @@ "program = \"default\"\n", "study_type = \"study\"\n", "include_test_users = False\n", - "dynamic_labels = {}\n", + "labels = {}\n", "use_imperial = True\n", "sensed_algo_prefix = \"cleaned\"\n", "bluetooth_only = True #current proxy for fleet status\n", @@ -61,7 +61,7 @@ "label_units, short_label, label_units_lower, distance_col, weight_unit = scaffolding.get_units(use_imperial)\n", "\n", "# get color mappings\n", - "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels() #just need sensed" + "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = scaffolding.mapping_color_labels() #just need sensed" ] }, { @@ -114,7 +114,7 @@ " expanded_ct_sensed[\"primary_mode\"] = expanded_ct_sensed.ble_sensed_summary.apply(lambda md: max(md[\"distance\"], key=md[\"distance\"].get))\n", " unique_keys = expanded_ct_sensed.groupby(\"primary_mode\").agg({distance_col: \"count\"}).index\n", " print(unique_keys)\n", - " colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(unique_keys)\n", + " colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = scaffolding.mapping_color_labels(unique_keys)\n", " colors_sensed = colors_ble\n", "except ValueError as e:\n", " print(\"Got ValueError \", e)" From 7815a10068e4971b8371d7924c2278b160e4c535 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 13:16:11 -0700 Subject: [PATCH 11/15] Replace dynamic_labels with labels. --- viz_scripts/survey_responses.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/survey_responses.ipynb b/viz_scripts/survey_responses.ipynb index 53a633c..6b045cc 100644 --- a/viz_scripts/survey_responses.ipynb +++ b/viz_scripts/survey_responses.ipynb @@ -13,7 +13,7 @@ "study_type = \"study\"\n", "mode_of_interest = None\n", "include_test_users = True\n", - "dynamic_labels = {}\n", + "labels = {}\n", "use_imperial = True\n", "survey_info = {}" ] From 2916df22e6906f43876812a80d5516ce36027443 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 13:20:09 -0700 Subject: [PATCH 12/15] Remove await from mapping_color_labels() as this is not an async function anymore. --- viz_scripts/generic_metrics_sensed.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/generic_metrics_sensed.ipynb b/viz_scripts/generic_metrics_sensed.ipynb index 9d4191f..aff60da 100644 --- a/viz_scripts/generic_metrics_sensed.ipynb +++ b/viz_scripts/generic_metrics_sensed.ipynb @@ -96,7 +96,7 @@ " expanded_ct[\"primary_mode\"] = expanded_ct.ble_sensed_summary.apply(lambda md: max(md[\"distance\"], key=md[\"distance\"].get))\n", " unique_keys = expanded_ct.groupby(\"primary_mode\").agg({distance_col: \"count\"}).index\n", " print(unique_keys)\n", - " colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(unique_keys) #Extract ble color mapping\n", + " colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = scaffolding.mapping_color_labels(unique_keys) #Extract ble color mapping\n", " colors_sensed = colors_ble\n", "except ValueError as e:\n", " print(\"Got ValueError \", e)" From 551c0cd359c42992f0ea107c04c7c48cda7dc4cc Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 16:23:14 -0700 Subject: [PATCH 13/15] Add labels as the parameter for load_viz_notebook_sensor_inference_data() --- viz_scripts/bin/commit_message.txt | 6 ++++++ viz_scripts/commit_message.txt | 8 ++++++++ viz_scripts/generic_metrics.ipynb | 1 + viz_scripts/generic_metrics_sensed.ipynb | 2 ++ viz_scripts/generic_timeseries.ipynb | 1 + viz_scripts/survey_metrics.ipynb | 1 + 6 files changed, 19 insertions(+) create mode 100644 viz_scripts/bin/commit_message.txt create mode 100644 viz_scripts/commit_message.txt diff --git a/viz_scripts/bin/commit_message.txt b/viz_scripts/bin/commit_message.txt new file mode 100644 index 0000000..d458fab --- /dev/null +++ b/viz_scripts/bin/commit_message.txt @@ -0,0 +1,6 @@ +Load read_json_resource() from e-mission-common + +- Pass labels instead of dynamic_labels to the notebooks +- labels can either be: + - Custom labels from the program deployment config, from nrel-openpath-deploy-configs + - In case, we do not have custom labels for the program/study, load default labels from e-mission-common with read_json_resource \ No newline at end of file diff --git a/viz_scripts/commit_message.txt b/viz_scripts/commit_message.txt new file mode 100644 index 0000000..2171fd9 --- /dev/null +++ b/viz_scripts/commit_message.txt @@ -0,0 +1,8 @@ +Remove call to read_json_resource("label-options.default.json") from e-mission-common in scaffolding.py + +Replace dynamic_labels with "labels" + - labels could be either default labels or dynamic labels (provided for program/study) depending on availability in config file + - Use labels passed from notebook instead of calling read_json_resource("label-options.default.json") from e-mission-common +Remove async and await for mapping_color_labels() and translate_values_to_labels() + Remove filtering for dynamic_labels availability, as we will only have either labels or dynamic_labels. +Update load_viz_notebook_* functions to use labels instead of dynamic_labels \ No newline at end of file diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 370d6d7..2cf6b18 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -142,6 +142,7 @@ "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", + " labels,\n", " include_test_users,\n", " sensed_algo_prefix)" ] diff --git a/viz_scripts/generic_metrics_sensed.ipynb b/viz_scripts/generic_metrics_sensed.ipynb index aff60da..a800c84 100644 --- a/viz_scripts/generic_metrics_sensed.ipynb +++ b/viz_scripts/generic_metrics_sensed.ipynb @@ -30,6 +30,7 @@ "program = \"default\"\n", "study_type = \"study\"\n", "mode_of_interest = None\n", + "labels = {}\n", "include_test_users = False\n", "use_imperial = False\n", "sensed_algo_prefix = \"cleaned\"\n", @@ -77,6 +78,7 @@ "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", + " labels,\n", " include_test_users,\n", " sensed_algo_prefix)" ] diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index 2171c9a..40d49d9 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -92,6 +92,7 @@ "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", + " labels,\n", " include_test_users,\n", " sensed_algo_prefix)" ] diff --git a/viz_scripts/survey_metrics.ipynb b/viz_scripts/survey_metrics.ipynb index 2ff91b6..cfa032d 100644 --- a/viz_scripts/survey_metrics.ipynb +++ b/viz_scripts/survey_metrics.ipynb @@ -96,6 +96,7 @@ "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", + " labels,\n", " include_test_users,\n", " sensed_algo_prefix)" ] From 76b4a129a0c92124f2e79439b2711d2f6315b65b Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Mon, 14 Oct 2024 12:27:10 -0700 Subject: [PATCH 14/15] Remove commit_message.txt files from the git commit. Add labels as the parameter for load_viz_notebook_sensor_inference_data(). --- viz_scripts/generic_metrics.ipynb | 1 + viz_scripts/generic_metrics_sensed.ipynb | 2 ++ viz_scripts/generic_timeseries.ipynb | 1 + viz_scripts/survey_metrics.ipynb | 1 + 4 files changed, 5 insertions(+) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 370d6d7..2cf6b18 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -142,6 +142,7 @@ "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", + " labels,\n", " include_test_users,\n", " sensed_algo_prefix)" ] diff --git a/viz_scripts/generic_metrics_sensed.ipynb b/viz_scripts/generic_metrics_sensed.ipynb index aff60da..a800c84 100644 --- a/viz_scripts/generic_metrics_sensed.ipynb +++ b/viz_scripts/generic_metrics_sensed.ipynb @@ -30,6 +30,7 @@ "program = \"default\"\n", "study_type = \"study\"\n", "mode_of_interest = None\n", + "labels = {}\n", "include_test_users = False\n", "use_imperial = False\n", "sensed_algo_prefix = \"cleaned\"\n", @@ -77,6 +78,7 @@ "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", + " labels,\n", " include_test_users,\n", " sensed_algo_prefix)" ] diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index 2171c9a..40d49d9 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -92,6 +92,7 @@ "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", + " labels,\n", " include_test_users,\n", " sensed_algo_prefix)" ] diff --git a/viz_scripts/survey_metrics.ipynb b/viz_scripts/survey_metrics.ipynb index 2ff91b6..cfa032d 100644 --- a/viz_scripts/survey_metrics.ipynb +++ b/viz_scripts/survey_metrics.ipynb @@ -96,6 +96,7 @@ "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", + " labels,\n", " include_test_users,\n", " sensed_algo_prefix)" ] From 74708a8949dd84c7ed22e7acc4022be8283d0e77 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Mon, 14 Oct 2024 12:34:25 -0700 Subject: [PATCH 15/15] Remove commit_message.txt files --- viz_scripts/bin/commit_message.txt | 6 ------ viz_scripts/commit_message.txt | 8 -------- 2 files changed, 14 deletions(-) delete mode 100644 viz_scripts/bin/commit_message.txt delete mode 100644 viz_scripts/commit_message.txt diff --git a/viz_scripts/bin/commit_message.txt b/viz_scripts/bin/commit_message.txt deleted file mode 100644 index d458fab..0000000 --- a/viz_scripts/bin/commit_message.txt +++ /dev/null @@ -1,6 +0,0 @@ -Load read_json_resource() from e-mission-common - -- Pass labels instead of dynamic_labels to the notebooks -- labels can either be: - - Custom labels from the program deployment config, from nrel-openpath-deploy-configs - - In case, we do not have custom labels for the program/study, load default labels from e-mission-common with read_json_resource \ No newline at end of file diff --git a/viz_scripts/commit_message.txt b/viz_scripts/commit_message.txt deleted file mode 100644 index 2171fd9..0000000 --- a/viz_scripts/commit_message.txt +++ /dev/null @@ -1,8 +0,0 @@ -Remove call to read_json_resource("label-options.default.json") from e-mission-common in scaffolding.py - -Replace dynamic_labels with "labels" - - labels could be either default labels or dynamic labels (provided for program/study) depending on availability in config file - - Use labels passed from notebook instead of calling read_json_resource("label-options.default.json") from e-mission-common -Remove async and await for mapping_color_labels() and translate_values_to_labels() - Remove filtering for dynamic_labels availability, as we will only have either labels or dynamic_labels. -Update load_viz_notebook_* functions to use labels instead of dynamic_labels \ No newline at end of file