diff --git a/README.md b/README.md index 7919d68..aa5a236 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,9 @@ https://towardsdatascience.com/dockerize-your-dash-app-1e155dd1cea3 ## How to run it: Docker Compose (recommended) -`docker compose -f docker-compose-dash-app.yml build` +`docker compose -f docker-compose-dev.yml build` -`docker compose -f docker-compose-dash-app.yml up` +`docker compose -f docker-compose-dev.yml up` You **must** use this method. Do **not** try to directly by setting up a virtualenv with the `requirements.txt` This uses components of the e-mission-server core, so it must have the e-mission-server modules in the PYTHONPATH @@ -65,12 +65,15 @@ These are all the permissions that you can specify: ### Data Page - `data_uuids`: User can view the UUIDs data in the Data page. - `data_trips`: User can view the trips data in the Data page. +- `data_demographics`: User can view the demographics data in the Data page. - `data_trips_columns_exclude`: It used to specify a list of column names that should be excluded from the trips data that is displayed on the Data page. It includes valid columns from the **Stage_analysis_timeseries** collection. Valid columns are specified in the following sections. - `data_uuids_columns_exclude`: It used to specify a list of column names that should be excluded from the uuids data that is displayed on the Data page. It includes valid columns from the **Stage_uuids** collection. Valid columns are specified in the following sections. +- `data_demographics_columns_exclude`: It used to specify a list of column names that should be excluded from the demographics data +that is displayed on the Data page. ### Token Page - `token_generate`: User can generate new tokens in the Token page. diff --git a/app_sidebar_collapsible.py b/app_sidebar_collapsible.py index 64077db..8380564 100644 --- a/app_sidebar_collapsible.py +++ b/app_sidebar_collapsible.py @@ -23,7 +23,7 @@ if os.getenv('DASH_DEBUG_MODE', 'True').lower() == 'true': logging.basicConfig(level=logging.DEBUG) -from utils.db_utils import query_uuids, query_confirmed_trips +from utils.db_utils import query_uuids, query_confirmed_trips, query_demographics from utils.permissions import has_permission import flask_talisman as flt @@ -157,12 +157,24 @@ content, ] +def update_store_demographics(): + df = query_demographics() + records = df.to_dict("records") + + store = { + "data": records, + "length": len(records), + } + return store + +demographics_data = update_store_demographics() app.layout = html.Div( [ dcc.Location(id='url', refresh=False), dcc.Store(id='store-trips', data={}), dcc.Store(id='store-uuids', data={}), + dcc.Store(id='store-demographics', data= demographics_data), html.Div(id='page-content', children=home_page), ] ) diff --git a/docker/load_mongodump.sh b/docker/load_mongodump.sh new file mode 100755 index 0000000..bd6fcea --- /dev/null +++ b/docker/load_mongodump.sh @@ -0,0 +1,10 @@ +MONGODUMP_FILE=$1 + +echo "Copying file to docker container" +docker cp $MONGODUMP_FILE op-admin-dashboard-db-1:/tmp + +FILE_NAME=`basename $MONGODUMP_FILE` + +echo "Restoring the dump from $FILE_NAME" +docker exec -e MONGODUMP_FILE=$FILE_NAME op-admin-dashboard-db-1 bash -c 'cd /tmp && tar xvf $MONGODUMP_FILE && mongorestore' + diff --git a/pages/data.py b/pages/data.py index 851c4fa..0f0be07 100644 --- a/pages/data.py +++ b/pages/data.py @@ -21,9 +21,9 @@ [ dcc.Markdown(intro), dcc.Tabs(id="tabs-datatable", value='tab-uuids-datatable', children=[ - # dcc.Tab(label='Demographics survey', value='tab-demographics-survey-datatable'), dcc.Tab(label='UUIDs', value='tab-uuids-datatable'), dcc.Tab(label='Trips', value='tab-trips-datatable'), + dcc.Tab(label='Demographics', value='tab-demographics-datatable' ) ]), html.Div(id='tabs-content'), ] @@ -43,8 +43,9 @@ def clean_location_data(df): Input('tabs-datatable', 'value'), Input('store-uuids', 'data'), Input('store-trips', 'data'), + Input('store-demographics', 'data'), ) -def render_content(tab, store_uuids, store_trips): +def render_content(tab, store_uuids, store_trips, store_demographics): data, columns, has_perm = None, [], False if tab == 'tab-uuids-datatable': data = store_uuids["data"] @@ -58,6 +59,11 @@ def render_content(tab, store_uuids, store_trips): col['label'] for col in perm_utils.get_allowed_named_trip_columns() ) has_perm = perm_utils.has_permission('data_trips') + elif tab == 'tab-demographics-datatable': + data = store_demographics["data"] + columns = list(data[0].keys()) + has_perm = perm_utils.has_permission('data_demographics') + df = pd.DataFrame(data) if df.empty or not has_perm: return None diff --git a/utils/constants.py b/utils/constants.py index 9eed848..56a3425 100644 --- a/utils/constants.py +++ b/utils/constants.py @@ -44,3 +44,8 @@ 'os_version', 'phone_lang', ] + +BINARY_DEMOGRAPHICS_COLS = [ + 'user_id', + '_id', +] \ No newline at end of file diff --git a/utils/db_utils.py b/utils/db_utils.py index 1500633..6c15eda 100644 --- a/utils/db_utils.py +++ b/utils/db_utils.py @@ -98,6 +98,24 @@ def query_confirmed_trips(start_date, end_date): # logging.debug("After filtering, the actual data is %s" % df.head().trip_start_time_str) return df +def query_demographics(): + ts = esta.TimeSeries.get_aggregate_time_series() + + entries = ts.find_entries(["manual/demographic_survey"]) + df = pd.json_normalize(list(entries)) + if not df.empty: + for col in constants.BINARY_DEMOGRAPHICS_COLS: + if col in df.columns: + df[col] = df[col].apply(str) + columns_to_drop = [col for col in df.columns if col.startswith("metadata")] + df.drop(columns= columns_to_drop, inplace=True) + df.drop(columns=['data.xmlResponse', 'data.name', 'data.version', 'data.label'], inplace=True) + modified_columns = perm_utils.get_demographic_columns(df.columns) + df.columns = modified_columns + df.columns=[col.rsplit('.',1)[-1] if col.startswith('data.jsonDocResponse.') else col for col in df.columns] + df.drop(columns=['xmlns:jr', 'xmlns:orx', 'id'], inplace = True) + return df + def add_user_stats(user_data): for user in user_data: diff --git a/utils/permissions.py b/utils/permissions.py index 99ec0d9..ffc49df 100644 --- a/utils/permissions.py +++ b/utils/permissions.py @@ -29,6 +29,10 @@ }) permissions = config.get("admin_dashboard", {}) +# TODO: The current dynamic config does not have the data_demographics_columns_exclude. +# When all the current studies are completed we can remove the below changes. +if 'data_demographics_columns_exclude' not in permissions: + permissions['data_demographics_columns_exclude'] = [] def has_permission(perm): return False if permissions.get(perm) is False else True @@ -87,6 +91,10 @@ def get_uuids_columns(): columns.discard(column) return columns +def get_demographic_columns(columns): + for column in permissions.get("data_demographics_columns_exclude", []): + columns.discard(column) + return columns def get_token_prefix(): return permissions['token_prefix'] + '_' if permissions.get('token_prefix') else ''