Merge branch 'image-push' into tags-combo-approach

frontend/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:21.5.0-alpine
+FROM node:21.7.2-alpine
 VOLUME /public/plots
 
 RUN mkdir -p /public

frontend/docker/Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM node:21.5.0-alpine
+FROM node:21.7.2-alpine
 
 RUN mkdir -p /public
 WORKDIR /public

viz_scripts/Dockerfile

@@ -1,8 +1,6 @@
 # python 3
-# TODO: Please change once all PR changes are final, so it reads from shankari/e-mission-server
-# FROM shankari/e-mission-server:master_2024-02-10--19-38
 ARG DOCKER_IMAGE_TAG
-FROM mukuflash03/e-mission-server:tags-combo-approach_${DOCKER_IMAGE_TAG}
+FROM shankari/e-mission-server:master_${DOCKER_IMAGE_TAG}
 
 VOLUME /plots
 

viz_scripts/docker/Dockerfile.dev

@@ -1,8 +1,7 @@
 # python 3
-# TODO: Please change once all PR changes are final, so it reads from shankari/e-mission-server
-# FROM shankari/e-mission-server:master_2024-02-10--19-38
+
 ARG DOCKER_IMAGE_TAG
-FROM mukuflash03/e-mission-server:tags-combo-approach_${DOCKER_IMAGE_TAG}
+FROM shankari/e-mission-server:master_${DOCKER_IMAGE_TAG}
 
 VOLUME /plots
 

viz_scripts/generic_metrics.ipynb

@@ -74,6 +74,24 @@
     "label_units, short_label, label_units_lower, distance_col, weight_unit = scaffolding.get_units(use_imperial)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "796f59c7",
+   "metadata": {},
+   "source": [
+    "### Color Mapping for labels"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "14e7260c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "colors_mode, colors_purpose = scaffolding.mapping_color_labels(dynamic_labels, dic_re, dic_pur)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "intellectual-columbus",
@@ -130,7 +148,7 @@
     "    labels_mc = expanded_ct['Mode_confirm'].value_counts(dropna=True).keys().tolist()\n",
     "    values_mc = expanded_ct['Mode_confirm'].value_counts(dropna=True).tolist()    \n",
     "    plot_title = plot_title_no_quality+\"\\n\"+quality_text\n",
-    "    pie_chart_mode(plot_title,labels_mc,values_mc,file_name)\n",
+    "    pie_chart_mode(plot_title,labels_mc,values_mc,colors_mode,file_name)\n",
     "    alt_text = store_alt_text_pie(pd.DataFrame(values_mc, labels_mc), file_name, plot_title)\n",
     "    print(expanded_ct['Mode_confirm'].value_counts(dropna=True))\n",
     "except:\n",
@@ -162,7 +180,8 @@
     "    values_mc = expanded_ct.query(trip_purpose_query).Mode_confirm.value_counts(dropna=True).tolist()\n",
     "    commute_quality_text = scaffolding.get_quality_text(expanded_ct, expanded_ct.query(trip_purpose_query), \"commute\", include_test_users)\n",
     "    plot_title= plot_title_no_quality+\"\\n\"+commute_quality_text\n",
-    "    pie_chart_mode(plot_title,labels_mc,values_mc,file_name)\n",
+    "    \n",
+    "    pie_chart_mode(plot_title,labels_mc,values_mc,colors_mode,file_name)\n",
     "    alt_text = store_alt_text_pie(pd.DataFrame(values_mc, labels_mc), file_name, plot_title)\n",
     "except:\n",
     "    debug_df.loc[\"Commute_trips\"] = len(expanded_ct.query(trip_purpose_query)) if \"Trip_purpose\" in expanded_ct.columns else 0\n",
@@ -194,7 +213,7 @@
     "    labels_tp = expanded_ct['Trip_purpose'].value_counts(dropna=True).keys().tolist()\n",
     "    values_tp = expanded_ct['Trip_purpose'].value_counts(dropna=True).tolist()\n",
     "    plot_title= plot_title_no_quality+\"\\n\"+quality_text\n",
-    "    pie_chart_purpose(plot_title,labels_tp,values_tp,file_name)\n",
+    "    pie_chart_purpose(plot_title,labels_tp,values_tp,colors_purpose,file_name)\n",
     "    alt_text = store_alt_text_pie(pd.DataFrame(values_tp, labels_tp), file_name, plot_title)\n",
     "    print(expanded_ct['Trip_purpose'].value_counts(dropna=True))\n",
     "except:\n",
@@ -236,7 +255,7 @@
     "    values_d10 = expanded_ct.loc[(expanded_ct['distance'] <= cutoff)].Mode_confirm.value_counts(dropna=True).tolist()\n",
     "    d10_quality_text = scaffolding.get_quality_text(expanded_ct, expanded_ct[expanded_ct['distance'] <= cutoff], \"< \" + dist_threshold + \" \" + label_units_lower, include_test_users)\n",
     "    plot_title = plot_title_no_quality+\"\\n\"+d10_quality_text\n",
-    "    pie_chart_mode(plot_title,labels_d10,values_d10,file_name)\n",
+    "    pie_chart_mode(plot_title,labels_d10,values_d10,colors_mode,file_name)\n",
     "    alt_text = store_alt_text_pie(pd.DataFrame(values_d10, labels_d10), file_name, plot_title)\n",
     "    print(expanded_ct.loc[(expanded_ct['distance'] <= cutoff)].Mode_confirm.value_counts(dropna=True))\n",
     "\n",
@@ -282,7 +301,7 @@
     "        labels_m.append(x)\n",
     "        values_m.append(y)\n",
     "\n",
-    "    pie_chart_mode(plot_title,labels_m,values_m,file_name)\n",
+    "    pie_chart_mode(plot_title,labels_m,values_m,colors_mode,file_name)\n",
     "    alt_text = store_alt_text_pie(pd.DataFrame(values_m, labels_m), file_name, plot_title)\n",
     "except:\n",
     "    generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n",

viz_scripts/mode_specific_metrics.ipynb

@@ -88,6 +88,24 @@
     "label_units, short_label, label_units_lower, distance_col, weight_unit = scaffolding.get_units(use_imperial)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "397709c8",
+   "metadata": {},
+   "source": [
+    "### Color Mapping for labels"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3caccda7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "colors_mode, colors_purpose = scaffolding.mapping_color_labels(dynamic_labels, dic_re, dic_pur)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "built-occupation",
@@ -178,7 +196,7 @@
     "    labels_tp = data_eb['Trip_purpose'].value_counts(dropna=True).keys().tolist()\n",
     "    values_tp = data_eb['Trip_purpose'].value_counts(dropna=True).tolist()\n",
     "    plot_title= plot_title_no_quality+\"\\n\"+quality_text\n",
-    "    pie_chart_purpose(plot_title,labels_tp,values_tp,file_name)\n",
+    "    pie_chart_purpose(plot_title,labels_tp,values_tp,colors_purpose,file_name)\n",
     "    alt_text = store_alt_text_pie(pd.DataFrame(values_tp, labels_tp), file_name, plot_title)\n",
     "except:\n",
     "    generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n",
@@ -199,7 +217,7 @@
     "    labels_eb = data_eb.Replaced_mode.value_counts(dropna=True).keys().tolist()\n",
     "    values_eb = data_eb.Replaced_mode.value_counts(dropna=True).tolist()\n",
     "    plot_title= plot_title_no_quality+\"\\n\"+quality_text\n",
-    "    pie_chart_mode(plot_title,labels_eb,values_eb,file_name)\n",
+    "    pie_chart_mode(plot_title,labels_eb,values_eb,colors_mode,file_name)\n",
     "    alt_text = store_alt_text_pie(pd.DataFrame(values_eb, labels_eb), file_name, plot_title)\n",
     "except:\n",
     "    generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n",
@@ -240,7 +258,7 @@
     "        values_m.append(y)\n",
     "\n",
     "    plot_title= plot_title_no_quality+\"\\n\"+quality_text\n",
-    "    pie_chart_mode(plot_title,labels_m,values_m,file_name)\n",
+    "    pie_chart_mode(plot_title,labels_m,values_m,colors_mode,file_name)\n",
     "    alt_text = store_alt_text_pie(pd.DataFrame(values_m, labels_m), file_name, plot_title)\n",
     "    print(dg)\n",
     "except:\n",

viz_scripts/plots.py

@@ -57,16 +57,13 @@ def format_pct(pct, values):
     return "{:.1f}%\n({:d})".format(pct, absolute) if pct > 4 else''
 
 
-def pie_chart_mode(plot_title,labels,values,file_name):
-
-    colours = dict(zip(labels, plt.cm.tab20.colors[:len(labels)]))
+def pie_chart_mode(plot_title,labels,values,colors_map,file_name):
     fig, ax = plt.subplots(figsize=(10, 10), subplot_kw=dict(aspect="equal"))
-
     m_labels, m_values = merge_small_entries(labels, values)
-    
+
     wedges, texts, autotexts = ax.pie(m_values,
                                       labels = m_labels,
-                                      colors=[colours[key] for key in labels],
+                                      colors=[colors_map[key] for key in labels],
                                       pctdistance=0.75,
                                       autopct= lambda pct: format_pct(pct, values),
                                       textprops={'size': 23})
@@ -105,9 +102,8 @@ def pie_chart_sensed_mode(plot_title,labels,values,file_name):
     plt.savefig(SAVE_DIR+file_name+".png", bbox_inches='tight')
     plt.show()
 
-def pie_chart_purpose(plot_title,labels,values,file_name):
+def pie_chart_purpose(plot_title,labels,values,colors_map,file_name):
 
-    colours = dict(zip(labels, plt.cm.tab20.colors[:len(labels)]))
     fig, ax = plt.subplots(figsize=(10, 10), subplot_kw=dict(aspect="equal"))
 
     m_labels, m_values = merge_small_entries(labels, values)
@@ -119,7 +115,7 @@ def func(pct, values):
 
     wedges, texts, autotexts = ax.pie(m_values,
                                       labels = m_labels,
-                                      colors=[colours[key] for key in labels],
+                                      colors=[colors_map[key] for key in labels],
                                       pctdistance=0.85,
                                       autopct=lambda pct: func(pct, values),
                                       textprops={'size': 23})

viz_scripts/scaffolding.py

@@ -1,7 +1,9 @@
 import pandas as pd
 import numpy as np
+import matplotlib.pyplot as plt
 import sys
 from collections import defaultdict
+from collections import OrderedDict
 
 import emission.storage.timeseries.abstract_timeseries as esta
 import emission.storage.timeseries.tcquery as esttc
@@ -192,6 +194,25 @@ def translate_labels(labels):
         dic_mapping = translate_labels(dynamic_labels[label_type])
         return dic_mapping
 
+# Function: Maps "MODE", "PURPOSE", and "REPLACED_MODE" to colors.
+# Input: dynamic_labels, dic_re, and dic_pur
+# Output: Map for color with mode and purpose
+def mapping_color_labels(dynamic_labels, dic_re, dic_pur):
+    if len(dynamic_labels) > 0:
+        mode_values = list(mapping_labels(dynamic_labels, "MODE").values()) if "MODE" in dynamic_labels else []
+        replaced_mode_values = list(mapping_labels(dynamic_labels, "REPLACED_MODE").values()) if "REPLACED_MODE" in dynamic_labels else []
+        purpose_values = list(mapping_labels(dynamic_labels, "PURPOSE").values()) if "PURPOSE" in dynamic_labels else []
+        combined_mode_values = mode_values + replaced_mode_values
+    else:
+        # Addition of 'Other' is required to the list since it's missing from auxillary_files/mode_labels.csv and auxillary_files/purpose_labels.csv
+        combined_mode_values = (list(OrderedDict.fromkeys(dic_re.values())) + ['Other'])
+        purpose_values = (list(OrderedDict.fromkeys(dic_pur.values())) + ['Other'])
+
+    colors_mode = dict(zip(combined_mode_values, plt.cm.tab20.colors[:len(combined_mode_values)]))
+    colors_purpose = dict(zip(purpose_values, plt.cm.tab20.colors[:len(purpose_values)]))
+
+    return colors_mode, colors_purpose
+
 def load_viz_notebook_sensor_inference_data(year, month, program, include_test_users=False, sensed_algo_prefix="cleaned"):
     """ Inputs:
     year/month/program = parameters from the visualization notebook