Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Unify calls to read json resource from e-mission-common in generate_plots.py #164

Open
wants to merge 16 commits into
base: main
Choose a base branch
from
Open

Unify calls to read json resource from e-mission-common in generate_plots.py #164

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
2d81075
Update the STUDY_CONFIG in docker-compose.dev.yml config file with th…
iantei Oct 2, 2024
242a7ea
Update comment for updating docker-compose config file with actual ST…
iantei Oct 2, 2024
4c05924
Load read_json_resource() from e-mission-common
iantei Oct 3, 2024
3b5b346
Remove call to read_json_resource("label-options.default.json") from …
iantei Oct 3, 2024
413420d
Replace dynamic_labels with labels. Remove await from mapping_color_l…
iantei Oct 3, 2024
db67977
Replace dynamic_labels with labels in energy_calculations notebook.
iantei Oct 3, 2024
ab70832
Replace dynamic_labels with labels in generic_timeseries notebook. Re…
iantei Oct 3, 2024
a050320
Replace dynamic_labels with labels in mode_specific_metrics notebook.…
iantei Oct 3, 2024
595876d
Replace dynamic_labels with labels. Remove check for dynamic_labels s…
iantei Oct 3, 2024
876bdc2
Replace dynamic_labels with labels. Remove await from mapping_color_l…
iantei Oct 3, 2024
7815a10
Replace dynamic_labels with labels.
iantei Oct 3, 2024
2916df2
Remove await from mapping_color_labels() as this is not an async func…
iantei Oct 3, 2024
551c0cd
Add labels as the parameter for load_viz_notebook_sensor_inference_da…
iantei Oct 3, 2024
76b4a12
Remove commit_message.txt files from the git commit. Add labels as th…
iantei Oct 14, 2024
12e3369
Merge branch 'unify_read_json_resource' of https://github.com/iantei/…
iantei Oct 14, 2024
74708a8
Remove commit_message.txt files
iantei Oct 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 16 additions & 5 deletions viz_scripts/bin/generate_plots.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
import json
import os
import sys

import asyncio
import emcommon.util as emcu

# Configuration settings to use for all generated plots by this instance
# This could also be specified as a parser argument, if we want to generate plots for all programs from one instance
Expand Down Expand Up @@ -43,7 +44,11 @@

# dynamic_labels can be referenced from
# https://github.com/e-mission/nrel-openpath-deploy-configs/blob/main/label_options/example-study-label-options.json
dynamic_labels = { }
labels = { }

async def load_default_label_options():
labels = await emcu.read_json_resource("label-options.default.json")
return labels

# Check if the dynamic config contains dynamic labels 'label_options'
# Parse through the dynamic_labels_url:
Expand All @@ -54,10 +59,16 @@
if req.status_code != 200:
print(f"Unable to download dynamic_labels_url, status code: {req.status_code} for {STUDY_CONFIG}")
else:
dynamic_labels = json.loads(req.text)
labels = json.loads(req.text)
print(f"Dynamic labels download was successful for nrel-openpath-deploy-configs: {STUDY_CONFIG}" )
else:
print(f"label_options is unavailable for the dynamic_config in {STUDY_CONFIG}")
# load default labels from e-mission-common
# https://raw.githubusercontent.com/JGreenlee/e-mission-common/refs/heads/master/src/emcommon/resources/label-options.default.json
labels = asyncio.run(load_default_label_options())
if not labels:
print(f"Unable to load labels for : {STUDY_CONFIG}")
else:
print(f"Labels loading was successful for nrel-openpath-deploy-configs: {STUDY_CONFIG}")

if args.date is None:
start_date = arrow.get(int(dynamic_config['intro']['start_year']),
Expand Down Expand Up @@ -88,7 +99,7 @@ def compute_for_date(month, year):
study_type=dynamic_config['intro']['program_or_study'],
mode_of_interest=mode_studied,
include_test_users=dynamic_config.get('metrics', {}).get('include_test_users', False),
dynamic_labels = dynamic_labels,
labels = labels,
use_imperial = dynamic_config.get('display_config', {}).get('use_imperial', True),
sensed_algo_prefix=dynamic_config.get('metrics', {}).get('sensed_algo_prefix', "cleaned"),
bluetooth_only = dynamic_config.get('tracking', {}).get('bluetooth_only', False),
Expand Down
4 changes: 4 additions & 0 deletions viz_scripts/docker/load_mongodump.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ echo "Database Name: $DB_NAME"
DB_HOST="mongodb://db/$DB_NAME"
sed -i.bak "s|DB_HOST=.*|DB_HOST=$DB_HOST|" "$CONFIG_FILE"

# Update the docker-compose configuration file with the actual STUDY_CONFIG
STUDY_CONFIG=$(echo "$DB_NAME" | sed -E 's/openpath_prod_(.*)$/\1/' | tr '_' '-')
sed -i.bak "s|STUDY_CONFIG=.*|STUDY_CONFIG=$STUDY_CONFIG|" "$CONFIG_FILE"

echo "Updated docker-compose file:"
cat "$CONFIG_FILE"

Expand Down
8 changes: 4 additions & 4 deletions viz_scripts/energy_calculations.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
"mode_of_interest = \"e-bike\"\n",
"include_test_users = False\n",
"is_debug_mode = False\n",
"dynamic_labels = {}\n",
"labels = {}\n",
"use_imperial = True"
]
},
Expand Down Expand Up @@ -91,7 +91,7 @@
" month,\n",
" program,\n",
" study_type,\n",
" dynamic_labels,\n",
" labels,\n",
" include_test_users=include_test_users,\n",
" add_footprint=True)\n",
"\n",
Expand Down Expand Up @@ -224,9 +224,9 @@
" if (is_debug_mode == True):\n",
" ebco2_debug = ebco2_compute(data_eb, weight_unit_debug)\n",
" if (weight_unit == 'kg'):\n",
" scaffolding.print_CO2_emission_calculations(data_eb, ebco2_debug, ebco2, dynamic_labels)\n",
" scaffolding.print_CO2_emission_calculations(data_eb, ebco2_debug, ebco2, labels)\n",
" else:\n",
" scaffolding.print_CO2_emission_calculations(data_eb, ebco2, ebco2_debug, dynamic_labels)\n",
" scaffolding.print_CO2_emission_calculations(data_eb, ebco2, ebco2_debug, labels)\n",
"\n",
"try:\n",
" if use_imperial:\n",
Expand Down
11 changes: 6 additions & 5 deletions viz_scripts/generic_metrics.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
"program = \"default\"\n",
"study_type = \"study\"\n",
"include_test_users = False\n",
"dynamic_labels = {}\n",
"labels = {}\n",
"use_imperial = True\n",
"sensed_algo_prefix = \"cleaned\"\n",
"survey_info = {}"
Expand Down Expand Up @@ -97,8 +97,8 @@
"metadata": {},
"outputs": [],
"source": [
"colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(dynamic_labels)\n",
"values_to_translations, value_to_translations_purpose, values_to_translations_replaced = await scaffolding.translate_values_to_labels(dynamic_labels)"
"colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = scaffolding.mapping_color_labels(labels)\n",
"values_to_translations, value_to_translations_purpose, values_to_translations_replaced = scaffolding.translate_values_to_labels(labels)"
]
},
{
Expand All @@ -120,7 +120,7 @@
" month,\n",
" program,\n",
" study_type,\n",
" dynamic_labels,\n",
" labels,\n",
" include_test_users=include_test_users)"
]
},
Expand All @@ -142,6 +142,7 @@
"expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n",
" month,\n",
" program,\n",
" labels,\n",
" include_test_users,\n",
" sensed_algo_prefix)"
]
Expand All @@ -165,7 +166,7 @@
" month,\n",
" program,\n",
" study_type,\n",
" dynamic_labels,\n",
" labels,\n",
" include_test_users=include_test_users)"
]
},
Expand Down
4 changes: 3 additions & 1 deletion viz_scripts/generic_metrics_sensed.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
"program = \"default\"\n",
"study_type = \"study\"\n",
"mode_of_interest = None\n",
"labels = {}\n",
"include_test_users = False\n",
"use_imperial = False\n",
"sensed_algo_prefix = \"cleaned\"\n",
Expand Down Expand Up @@ -77,6 +78,7 @@
"expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n",
" month,\n",
" program,\n",
" labels,\n",
" include_test_users,\n",
" sensed_algo_prefix)"
]
Expand All @@ -96,7 +98,7 @@
" expanded_ct[\"primary_mode\"] = expanded_ct.ble_sensed_summary.apply(lambda md: max(md[\"distance\"], key=md[\"distance\"].get))\n",
" unique_keys = expanded_ct.groupby(\"primary_mode\").agg({distance_col: \"count\"}).index\n",
" print(unique_keys)\n",
" colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(unique_keys) #Extract ble color mapping\n",
" colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = scaffolding.mapping_color_labels(unique_keys) #Extract ble color mapping\n",
" colors_sensed = colors_ble\n",
"except ValueError as e:\n",
" print(\"Got ValueError \", e)"
Expand Down
9 changes: 3 additions & 6 deletions viz_scripts/generic_timeseries.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
"study_type = \"study\"\n",
"mode_of_interest = None\n",
"include_test_users = False\n",
"dynamic_labels = {}\n",
"labels = {}\n",
"use_imperial = False\n",
"sensed_algo_prefix = \"cleaned\""
]
Expand Down Expand Up @@ -84,14 +84,15 @@
" month,\n",
" program,\n",
" study_type,\n",
" dynamic_labels,\n",
" labels,\n",
" include_test_users=include_test_users,\n",
" add_footprint=True)\n",
"expanded_ct = scaffolding.unpack_energy_emissions(expanded_ct) if \"mode_confirm_footprint\" in expanded_ct.columns else expanded_ct\n",
"\n",
"expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n",
" month,\n",
" program,\n",
" labels,\n",
" include_test_users,\n",
" sensed_algo_prefix)"
]
Expand Down Expand Up @@ -143,10 +144,6 @@
" expanded_ct_sensed.user_id = pd.Categorical(expanded_ct_sensed.user_id)\n",
" expanded_ct_sensed.date_time = pd.Categorical(expanded_ct_sensed.date_time)\n",
"\n",
"if len(dynamic_labels) > 0:\n",
" labels=dynamic_labels\n",
"else:\n",
" labels = await emcu.read_json_resource(\"label-options.default.json\")\n",
"dic_mode_mapping = scaffolding.mapping_labels(labels, \"MODE\")\n",
"\n",
"if \"Mode_confirm\" in expanded_ct.columns:\n",
Expand Down
10 changes: 5 additions & 5 deletions viz_scripts/mode_specific_metrics.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
"study_type = \"program\"\n",
"mode_of_interest = \"e-bike\"\n",
"include_test_users = False\n",
"dynamic_labels = { }\n",
"labels = { }\n",
"use_imperial = True"
]
},
Expand Down Expand Up @@ -95,8 +95,8 @@
"metadata": {},
"outputs": [],
"source": [
"colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(dynamic_labels)\n",
"values_to_translations, value_to_translations_purpose, value_to_translations_replaced = await scaffolding.translate_values_to_labels(dynamic_labels)"
"colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = scaffolding.mapping_color_labels(labels)\n",
"values_to_translations, value_to_translations_purpose, value_to_translations_replaced = scaffolding.translate_values_to_labels(labels)"
]
},
{
Expand All @@ -118,7 +118,7 @@
" month,\n",
" program,\n",
" study_type,\n",
" dynamic_labels,\n",
" labels,\n",
" include_test_users=include_test_users)"
]
},
Expand All @@ -141,7 +141,7 @@
" month,\n",
" program,\n",
" study_type,\n",
" dynamic_labels,\n",
" labels,\n",
" include_test_users=include_test_users)"
]
},
Expand Down
8 changes: 2 additions & 6 deletions viz_scripts/mode_specific_timeseries.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
"study_type = \"program\"\n",
"mode_of_interest = \"e-bike\"\n",
"include_test_users = False\n",
"dynamic_labels = { }\n",
"labels = { }\n",
"use_imperial = False"
]
},
Expand Down Expand Up @@ -94,7 +94,7 @@
" month,\n",
" program,\n",
" study_type,\n",
" dynamic_labels,\n",
" labels,\n",
" include_test_users=include_test_users)"
]
},
Expand Down Expand Up @@ -177,10 +177,6 @@
" mode_distance_interest = mode_distance[mode_distance['mode_confirm']==mode_of_interest].copy()\n",
"\n",
" # Mapping new mode labels with dictionaries\n",
" if len(dynamic_labels) > 0:\n",
" labels=dynamic_labels\n",
" else:\n",
" labels = await emcu.read_json_resource(\"label-options.default.json\")\n",
" dic_mode_mapping = scaffolding.mapping_labels(labels, \"MODE\")\n",
" mode_counts['Mode_confirm'] = mode_counts['mode_confirm'].map(dic_mode_mapping)\n",
" mode_counts_interest['Mode_confirm'] = mode_counts_interest['mode_confirm'].map(dic_mode_mapping)\n",
Expand Down
Loading