invalid data is INVALID, sensed other as OTHER not Other

viz_scripts/plots.py

@@ -44,18 +44,35 @@ def merge_small_entries(labels, values):
     v2l_df = v2l_df.drop(small_chunk.index)
     disp.display(v2l_df)
 
-    # This part if a bit tricky
-    # We could have already had a non-zero other, and it could be small or large
-    if "Other" not in v2l_df.index:
-        # zero other will end up with misc_count
-        if misc_count.vals > 0:
+    # we need to let the sensed be "OTHER" and labeled or inferred be "Other"
+    # going to use capitalization as a flag
+    if v2l_df.index[0][-1].isupper():
+        print("Found uppercase last letter")
+        # This part if a bit tricky
+        # We could have already had a non-zero other, and it could be small or large
+        if "OTHER" not in v2l_df.index:
+            # zero other will end up with misc_count
+            if misc_count.vals > 0:
+                v2l_df.loc["OTHER"] = misc_count
+        elif "OTHER" in small_chunk.index:
+            # non-zero small other will already be in misc_count
+            v2l_df.loc["OTHER"] = misc_count
+        else:
+            # non-zero large other, will not already be in misc_count
+            v2l_df.loc["OTHER"] = v2l_df.loc["OTHER"] + misc_count
+    else: #assuming labeled or inferred
+        # This part if a bit tricky
+        # We could have already had a non-zero other, and it could be small or large
+        if "Other" not in v2l_df.index:
+            # zero other will end up with misc_count
+            if misc_count.vals > 0:
+                v2l_df.loc["Other"] = misc_count
+        elif "Other" in small_chunk.index:
+            # non-zero small other will already be in misc_count
             v2l_df.loc["Other"] = misc_count
-    elif "Other" in small_chunk.index:
-        # non-zero small other will already be in misc_count
-        v2l_df.loc["Other"] = misc_count
-    else:
-        # non-zero large other, will not already be in misc_count
-        v2l_df.loc["Other"] = v2l_df.loc["Other"] + misc_count
+        else:
+            # non-zero large other, will not already be in misc_count
+            v2l_df.loc["Other"] = v2l_df.loc["Other"] + misc_count
 
     disp.display(v2l_df)
 

viz_scripts/scaffolding.py

@@ -198,7 +198,7 @@ def translate_labels(labels):
 # Input: dynamic_labels, dic_re, and dic_pur
 # Output: Dictionary mapping between color with mode/purpose/sensed
 def mapping_color_labels(dynamic_labels, dic_re, dic_pur):
-    sensed_values = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "OTHER", "Other"]
+    sensed_values = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "OTHER", "INVALID"]
     if len(dynamic_labels) > 0:
         mode_values = list(mapping_labels(dynamic_labels, "MODE").values()) if "MODE" in dynamic_labels else []
         replaced_mode_values = list(mapping_labels(dynamic_labels, "REPLACED_MODE").values()) if "REPLACED_MODE" in dynamic_labels else []
@@ -241,7 +241,7 @@ def load_viz_notebook_sensor_inference_data(year, month, program, include_test_u
 
     #TODO-this is also in the admin dash, can we unify?
     get_max_mode_from_summary = lambda md: (
-            "UNKNOWN"
+            "INVALID"
             if not isinstance(md, dict)
             or "distance" not in md
             or not isinstance(md["distance"], dict)
@@ -252,14 +252,14 @@ def load_viz_notebook_sensor_inference_data(year, month, program, include_test_u
                 # Otherwise, return "INVALID".
                 max(md["distance"], key=md["distance"].get)
                 if len(md["distance"]) > 0
-                else "UNKNOWN"
+                else "INVALID"
             )
         )
 
     if len(expanded_ct) > 0:
         expanded_ct["primary_mode_non_other"] = participant_ct_df.cleaned_section_summary.apply(get_max_mode_from_summary)
         expanded_ct.primary_mode_non_other.replace({"ON_FOOT": "WALKING"}, inplace=True)
-        valid_sensed_modes = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN"]
+        valid_sensed_modes = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "INVALID"]
         expanded_ct["primary_mode"] = expanded_ct.primary_mode_non_other.apply(lambda pm: "OTHER" if pm not in valid_sensed_modes else pm)
 
     # Change meters to miles