Merge branch 'master' of https://github.com/e-mission/op-admin-dashboard

 into segment_trip_time_resolved_conflicts

README.md

@@ -5,9 +5,9 @@ https://towardsdatascience.com/dockerize-your-dash-app-1e155dd1cea3
 
 ## How to run it: Docker Compose (recommended)
 
-`docker compose -f docker-compose-dash-app.yml build`
+`docker compose -f docker-compose-dev.yml build`
 
-`docker compose -f docker-compose-dash-app.yml up`
+`docker compose -f docker-compose-dev.yml up`
 
 You **must** use this method. Do **not** try to directly by setting up a virtualenv with the `requirements.txt`
 This uses components of the e-mission-server core, so it must have the e-mission-server modules in the PYTHONPATH
@@ -65,12 +65,18 @@ These are all the permissions that you can specify:
 ### Data Page
 - `data_uuids`: User can view the UUIDs data in the Data page.
 - `data_trips`: User can view the trips data in the Data page.
+- `data_demographics`: User can view the demographics data in the Data page.
+- `data_trajectories`: User can view the trajectories data in the Data page.
 - `data_trips_columns_exclude`: It used to specify a list of column names that should be excluded from the trips data
 that is displayed on the Data page. It includes valid columns from the **Stage_analysis_timeseries** collection. Valid
 columns are specified in the following sections.
 - `data_uuids_columns_exclude`: It used to specify a list of column names that should be excluded from the uuids data
 that is displayed on the Data page. It includes valid columns from the **Stage_uuids** collection. Valid columns are 
 specified in the following sections.
+- `data_demographics_columns_exclude`: It used to specify a list of column names that should be excluded from the demographics data
+that is displayed on the Data page. 
+- `data_trajectories_columns_exclude`: It used to specify a list of column names that should be excluded from the trajectories data
+that is displayed on the Data page.
 
 ### Token Page
 - `token_generate`: User can generate new tokens in the Token page.

app_sidebar_collapsible.py

@@ -10,7 +10,7 @@
 For more details on building multi-page Dash applications, check out the Dash documentation: https://dash.plot.ly/urls
 """
 import os
-from datetime import date
+import arrow
 
 import dash
 import dash_bootstrap_components as dbc
@@ -23,7 +23,8 @@
 if os.getenv('DASH_DEBUG_MODE', 'True').lower() == 'true':
     logging.basicConfig(level=logging.DEBUG)
 
-from utils.db_utils import query_uuids, query_confirmed_trips
+from utils.datetime_utils import iso_to_date_only
+from utils.db_utils import df_to_filtered_records, query_uuids, query_confirmed_trips, query_demographics
 from utils.permissions import has_permission
 import flask_talisman as flt
 
@@ -133,61 +134,161 @@
     className="sidebar",
 )
 
+# Global controls including date picker and timezone selector
+def make_controls():
+  # according to docs, DatePickerRange will accept YYYY-MM-DD format
+  today_date = arrow.now().format('YYYY-MM-DD')
+  last_week_date = arrow.now().shift(days=-7).format('YYYY-MM-DD')
+  tomorrow_date = arrow.now().shift(days=1).format('YYYY-MM-DD')
+  return html.Div([
+      html.Div([
+          # Global Date Picker
+          dcc.DatePickerRange(
+              id='date-picker',
+              display_format='D MMM Y',
+              start_date=last_week_date,
+              end_date=today_date,
+              min_date_allowed='2010-1-1',
+              max_date_allowed=tomorrow_date,
+              initial_visible_month=today_date,
+          ),
+          dbc.Button(
+              html.I(className="fas fa-bars", id='collapse-icon'),
+              outline=True,
+              id="collapse-button",
+              n_clicks=0,
+              style={'color': '#444', 'border': '1px solid #dbdbdb',
+                    'border-radius': '3px', 'margin-left': '3px'}
+          ),
+      ],
+          style={'display': 'flex'},
+      ),
+      dbc.Collapse([
+          html.Div([
+              html.Span('Query trips using: ', style={'margin-right': '10px'}),
+              dcc.Dropdown(
+                  id='date-picker-timezone',
+                  options=[
+                      {'label': 'UTC Time', 'value': 'utc'},
+                      {'label': 'My Local Timezone', 'value': 'local'},
+                      # {'label': 'Local Timezone of Trips', 'value': 'trips'},
+                  ],
+                  value='utc',
+                  clearable=False,
+                  searchable=False,
+                  style={'width': '180px'},
+              )]
+          ),
 
-content = html.Div([
-    # Global Date Picker
-    html.Div(
-        dcc.DatePickerRange(
-            id='date-picker',
-            display_format='D/M/Y',
-            start_date_placeholder_text='D/M/Y',
-            end_date_placeholder_text='D/M/Y',
-            min_date_allowed=date(2010, 1, 1),
-            max_date_allowed=date.today(),
-            initial_visible_month=date.today(),
-        ), style={'margin': '10px 10px 0 0', 'display': 'flex', 'justify-content': 'right'}
-    ),
-
-    # Pages Content
-    dcc.Loading(
-        type='default',
-        fullscreen=True,
-        children=html.Div(dash.page_container, style={
-            "margin-left": "5rem",
-            "margin-right": "2rem",
-            "padding": "2rem 1rem",
-        })
-    ),
-])
+          dcc.Checklist(
+              id='global-filters',
+              options=[
+                  {'label': 'Exclude "test" users',
+                   'value': 'exclude-test-users'},
+              ],
+              value=['exclude-test-users'],
+              style={'margin-top': '10px'},
+          ),
+      ],
+          id='collapse-filters',
+          is_open=False,
+          style={'padding': '5px 15px 10px', 'border': '1px solid #dbdbdb', 'border-top': '0'}
+      ),
+  ],
+      style={'margin': '10px 10px 0 auto',
+             'width': 'fit-content',
+             'display': 'flex',
+             'flex-direction': 'column'}
+  )
+
+page_content = dcc.Loading(
+    type='default',
+    fullscreen=True,
+    children=html.Div(dash.page_container, style={
+        "margin-left": "5rem",
+        "margin-right": "2rem",
+        "padding": "2rem 1rem",
+    })
+)
 
 
-home_page = [
+def make_home_page(): return [
     sidebar,
-    content,
+    html.Div([make_controls(), page_content])
 ]
 
 
-app.layout = html.Div(
-    [
-        dcc.Location(id='url', refresh=False),
-        dcc.Store(id='store-trips', data={}),
-        dcc.Store(id='store-uuids', data={}),
-        html.Div(id='page-content', children=home_page),
-    ]
-)
+def make_layout(): return html.Div([
+    dcc.Location(id='url', refresh=False),
+    dcc.Store(id='store-trips', data={}),
+    dcc.Store(id='store-uuids', data={}),
+    dcc.Store(id='store-excluded-uuids', data={}), # if 'test' users are excluded, a list of their uuids
+    dcc.Store(id='store-demographics', data={}),
+    dcc.Store(id='store-trajectories', data={}),
+    html.Div(id='page-content', children=make_home_page()),
+])
+app.layout = make_layout
 
+# make the 'filters' menu collapsible
+@app.callback(
+    Output("collapse-filters", "is_open"),
+    Output("collapse-icon", "className"),
+    [Input("collapse-button", "n_clicks")],
+    [Input("collapse-filters", "is_open")],
+)
+def toggle_collapse_filters(n, is_open):
+    if not n: return (is_open, "fas fa-bars")
+    if is_open:
+      return (False, "fas fa-bars")
+    else:
+      return (True, "fas fa-chevron-up")
 
 # Load data stores
 @app.callback(
     Output("store-uuids", "data"),
+    Output("store-excluded-uuids", "data"),
+    Input('date-picker', 'start_date'),  # these are ISO strings
+    Input('date-picker', 'end_date'),  # these are ISO strings
+    Input('date-picker-timezone', 'value'),
+    Input('global-filters', 'value'),
+)
+def update_store_uuids(start_date, end_date, timezone, filters):
+    (start_date, end_date) = iso_to_date_only(start_date, end_date)
+    dff = query_uuids(start_date, end_date, timezone)
+    if dff.empty:
+        return {"data": [], "length": 0}, {"data": [], "length": 0}
+    # if 'exclude-testusers' filter is active,
+    # exclude any rows with user_token containing 'test', and
+    # output a list of those excluded UUIDs so other callbacks can exclude them too
+    if 'exclude-test-users' in filters:
+        excluded_uuids_list = dff[dff['user_token'].str.contains(
+            'test')]['user_id'].tolist()
+    else:
+        excluded_uuids_list = []
+    records = df_to_filtered_records(dff, 'user_id', excluded_uuids_list)
+    store_uuids = {
+        "data": records,
+        "length": len(records),
+    }
+    store_excluded_uuids = {
+        "data": excluded_uuids_list,
+        "length": len(excluded_uuids_list),
+    }
+    return store_uuids, store_excluded_uuids
+
+
+@app.callback(
+    Output("store-demographics", "data"),
     Input('date-picker', 'start_date'),
     Input('date-picker', 'end_date'),
+    Input('date-picker-timezone', 'value'),
+    Input('store-excluded-uuids', 'data'),
 )
-def update_store_uuids(start_date, end_date):
-    start_date_obj = date.fromisoformat(start_date) if start_date else None
-    end_date_obj = date.fromisoformat(end_date) if end_date else None
-    dff = query_uuids(start_date_obj, end_date_obj)
-    records = dff.to_dict("records")
+def update_store_demographics(start_date, end_date, timezone, excluded_uuids):
+    dataframes = query_demographics()
+    records = {}
+    for key, df in dataframes.items():
+        records[key] = df_to_filtered_records(df, 'user_id', excluded_uuids["data"])
     store = {
         "data": records,
         "length": len(records),
@@ -198,14 +299,15 @@ def update_store_uuids(start_date, end_date):
 # Note: this triggers twice on load, not great with a slow db
 @app.callback(
     Output("store-trips", "data"),
-    Input('date-picker', 'start_date'),
-    Input('date-picker', 'end_date'),
+    Input('date-picker', 'start_date'), # these are ISO strings
+    Input('date-picker', 'end_date'), # these are ISO strings
+    Input('date-picker-timezone', 'value'),
+    Input('store-excluded-uuids', 'data'),
 )
-def update_store_trips(start_date, end_date):
-    start_date_obj = date.fromisoformat(start_date) if start_date else None
-    end_date_obj = date.fromisoformat(end_date) if end_date else None
-    df = query_confirmed_trips(start_date_obj, end_date_obj)
-    records = df.to_dict("records")
+def update_store_trips(start_date, end_date, timezone, excluded_uuids):
+    (start_date, end_date) = iso_to_date_only(start_date, end_date)
+    df = query_confirmed_trips(start_date, end_date, timezone)
+    records = df_to_filtered_records(df, 'user_id', excluded_uuids["data"])
     # logging.debug("returning records %s" % records[0:2])
     store = {
         "data": records,
@@ -228,10 +330,10 @@ def display_page(search):
             return get_cognito_login_page('Unsuccessful authentication, try again.', 'red')
 
         if is_authenticated:
-            return home_page
+            return make_home_page()
         return get_cognito_login_page()
 
-    return home_page
+    return make_home_page()
 
 extra_csp_url = [
     "https://raw.githubusercontent.com",

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM shankari/e-mission-server:gis-based-mode-detection_2023-04-21--54-09
+FROM shankari/e-mission-server:master_2024-02-10--19-38
 
 ENV DASH_DEBUG_MODE True
 ENV SERVER_PORT 8050

docker/load_mongodump.sh

@@ -0,0 +1,10 @@
+MONGODUMP_FILE=$1
+
+echo "Copying file to docker container"
+docker cp $MONGODUMP_FILE op-admin-dashboard-db-1:/tmp
+
+FILE_NAME=`basename $MONGODUMP_FILE`
+
+echo "Restoring the dump from $FILE_NAME"
+docker exec -e MONGODUMP_FILE=$FILE_NAME op-admin-dashboard-db-1 bash -c 'cd /tmp && tar xvf $MONGODUMP_FILE && mongorestore'
+