Merge pull request #63 from achasmita/demographics-table

Updated readme and added demographics table

README.md

@@ -5,9 +5,9 @@ https://towardsdatascience.com/dockerize-your-dash-app-1e155dd1cea3
 
 ## How to run it: Docker Compose (recommended)
 
-`docker compose -f docker-compose-dash-app.yml build`
+`docker compose -f docker-compose-dev.yml build`
 
-`docker compose -f docker-compose-dash-app.yml up`
+`docker compose -f docker-compose-dev.yml up`
 
 You **must** use this method. Do **not** try to directly by setting up a virtualenv with the `requirements.txt`
 This uses components of the e-mission-server core, so it must have the e-mission-server modules in the PYTHONPATH
@@ -65,12 +65,15 @@ These are all the permissions that you can specify:
 ### Data Page
 - `data_uuids`: User can view the UUIDs data in the Data page.
 - `data_trips`: User can view the trips data in the Data page.
+- `data_demographics`: User can view the demographics data in the Data page.
 - `data_trips_columns_exclude`: It used to specify a list of column names that should be excluded from the trips data
 that is displayed on the Data page. It includes valid columns from the **Stage_analysis_timeseries** collection. Valid
 columns are specified in the following sections.
 - `data_uuids_columns_exclude`: It used to specify a list of column names that should be excluded from the uuids data
 that is displayed on the Data page. It includes valid columns from the **Stage_uuids** collection. Valid columns are 
 specified in the following sections.
+- `data_demographics_columns_exclude`: It used to specify a list of column names that should be excluded from the demographics data
+that is displayed on the Data page. 
 
 ### Token Page
 - `token_generate`: User can generate new tokens in the Token page.

app_sidebar_collapsible.py

@@ -23,7 +23,7 @@
 if os.getenv('DASH_DEBUG_MODE', 'True').lower() == 'true':
     logging.basicConfig(level=logging.DEBUG)
 
-from utils.db_utils import query_uuids, query_confirmed_trips
+from utils.db_utils import query_uuids, query_confirmed_trips, query_demographics
 from utils.permissions import has_permission
 import flask_talisman as flt
 
@@ -157,12 +157,24 @@
     content,
 ]
 
+def update_store_demographics():
+    df = query_demographics()
+    records = df.to_dict("records")
+
+    store = {
+        "data": records,
+        "length": len(records),
+    }
+    return store
+
+demographics_data = update_store_demographics()
 
 app.layout = html.Div(
     [
         dcc.Location(id='url', refresh=False),
         dcc.Store(id='store-trips', data={}),
         dcc.Store(id='store-uuids', data={}),
+        dcc.Store(id='store-demographics', data= demographics_data),
         html.Div(id='page-content', children=home_page),
     ]
 )

docker/load_mongodump.sh

@@ -0,0 +1,10 @@
+MONGODUMP_FILE=$1
+
+echo "Copying file to docker container"
+docker cp $MONGODUMP_FILE op-admin-dashboard-db-1:/tmp
+
+FILE_NAME=`basename $MONGODUMP_FILE`
+
+echo "Restoring the dump from $FILE_NAME"
+docker exec -e MONGODUMP_FILE=$FILE_NAME op-admin-dashboard-db-1 bash -c 'cd /tmp && tar xvf $MONGODUMP_FILE && mongorestore'
+

pages/data.py

@@ -21,9 +21,9 @@
     [   
         dcc.Markdown(intro),
         dcc.Tabs(id="tabs-datatable", value='tab-uuids-datatable', children=[
-            # dcc.Tab(label='Demographics survey', value='tab-demographics-survey-datatable'),
             dcc.Tab(label='UUIDs', value='tab-uuids-datatable'),
             dcc.Tab(label='Trips', value='tab-trips-datatable'),
+            dcc.Tab(label='Demographics', value='tab-demographics-datatable' )
         ]),
         html.Div(id='tabs-content'),
     ]
@@ -43,8 +43,9 @@ def clean_location_data(df):
     Input('tabs-datatable', 'value'),
     Input('store-uuids', 'data'),
     Input('store-trips', 'data'),
+    Input('store-demographics', 'data'),
 )
-def render_content(tab, store_uuids, store_trips):
+def render_content(tab, store_uuids, store_trips, store_demographics):
     data, columns, has_perm = None, [], False
     if tab == 'tab-uuids-datatable':
         data = store_uuids["data"]
@@ -58,6 +59,11 @@ def render_content(tab, store_uuids, store_trips):
             col['label'] for col in perm_utils.get_allowed_named_trip_columns()
         )
         has_perm = perm_utils.has_permission('data_trips')
+    elif tab == 'tab-demographics-datatable':
+        data = store_demographics["data"]
+        columns = list(data[0].keys())
+        has_perm = perm_utils.has_permission('data_demographics')
+
     df = pd.DataFrame(data)
     if df.empty or not has_perm:
         return None

utils/constants.py

@@ -44,3 +44,8 @@
     'os_version',
     'phone_lang',
 ]
+
+BINARY_DEMOGRAPHICS_COLS = [
+    'user_id',
+    '_id',
+]

utils/db_utils.py

@@ -98,6 +98,24 @@ def query_confirmed_trips(start_date, end_date):
     # logging.debug("After filtering, the actual data is %s" % df.head().trip_start_time_str)
     return df
 
+def query_demographics():
+    ts = esta.TimeSeries.get_aggregate_time_series()
+
+    entries = ts.find_entries(["manual/demographic_survey"])
+    df = pd.json_normalize(list(entries))
+    if not df.empty:
+        for col in constants.BINARY_DEMOGRAPHICS_COLS:
+            if col in df.columns:
+                df[col] = df[col].apply(str) 
+    columns_to_drop = [col for col in df.columns if col.startswith("metadata")]
+    df.drop(columns= columns_to_drop, inplace=True) 
+    df.drop(columns=['data.xmlResponse', 'data.name', 'data.version', 'data.label'], inplace=True) 
+    modified_columns = perm_utils.get_demographic_columns(df.columns)  
+    df.columns = modified_columns 
+    df.columns=[col.rsplit('.',1)[-1] if col.startswith('data.jsonDocResponse.') else col for col in df.columns]  
+    df.drop(columns=['xmlns:jr', 'xmlns:orx', 'id'], inplace = True) 
+    return df
+
 
 def add_user_stats(user_data):
     for user in user_data:

utils/permissions.py

@@ -29,6 +29,10 @@
     })
 permissions = config.get("admin_dashboard", {})
 
+# TODO: The current dynamic config does not have the data_demographics_columns_exclude.
+# When all the current studies are completed we can remove the below changes.
+if 'data_demographics_columns_exclude' not in permissions:
+    permissions['data_demographics_columns_exclude'] = []
 
 def has_permission(perm):
     return False if permissions.get(perm) is False else True
@@ -87,6 +91,10 @@ def get_uuids_columns():
         columns.discard(column)
     return columns
 
+def get_demographic_columns(columns):
+    for column in permissions.get("data_demographics_columns_exclude", []):
+        columns.discard(column)
+    return columns
 
 def get_token_prefix():
     return permissions['token_prefix'] + '_' if permissions.get('token_prefix') else ''