diff --git a/pages/data.py b/pages/data.py index e0deb56..e4f220e 100644 --- a/pages/data.py +++ b/pages/data.py @@ -4,7 +4,7 @@ The workaround is to check if the input value is None. """ from dash import dcc, html, Input, Output, callback, register_page, dash_table - +from datetime import date # Etc import logging import pandas as pd @@ -38,9 +38,9 @@ def clean_location_data(df): df['data.end_loc.coordinates'] = df['data.end_loc.coordinates'].apply(lambda x: f'({x[0]}, {x[1]})') return df -def update_store_trajectories(): +def update_store_trajectories(start_date_obj,end_date_obj): global store_trajectories - df = query_trajectories() + df = query_trajectories(start_date_obj,end_date_obj) records = df.to_dict("records") store = { "data": records, @@ -57,8 +57,11 @@ def update_store_trajectories(): Input('store-trips', 'data'), Input('store-demographics', 'data'), Input('store-trajectories', 'data'), + Input('date-picker', 'start_date'), + Input('date-picker', 'end_date'), + ) -def render_content(tab, store_uuids, store_trips, store_demographics, store_trajectories): +def render_content(tab, store_uuids, store_trips, store_demographics, store_trajectories, start_date, end_date): data, columns, has_perm = None, [], False if tab == 'tab-uuids-datatable': data = store_uuids["data"] @@ -79,11 +82,14 @@ def render_content(tab, store_uuids, store_trips, store_demographics, store_traj elif tab == 'tab-trajectories-datatable': # Currently store_trajectories data is loaded only when the respective tab is selected #Here we query for trajectory data once "Trajectories" tab is selected + start_date_obj = date.fromisoformat(start_date) if start_date else None + end_date_obj = date.fromisoformat(end_date) if end_date else None if store_trajectories == {}: - store_trajectories = update_store_trajectories() + store_trajectories = update_store_trajectories(start_date_obj,end_date_obj) data = store_trajectories["data"] - columns = list(data[0].keys()) - has_perm = perm_utils.has_permission('data_trajectories') + if data: + columns = list(data[0].keys()) + has_perm = perm_utils.has_permission('data_trajectories') df = pd.DataFrame(data) if df.empty or not has_perm: diff --git a/utils/constants.py b/utils/constants.py index 5bc02e9..6cc197b 100644 --- a/utils/constants.py +++ b/utils/constants.py @@ -50,14 +50,6 @@ '_id', ] - -BINARY_TRAJECTORIES_COLS = [ - 'user_id', - '_id', - 'data.section', - 'data.loc.coordinates' -] - EXCLUDED_DEMOGRAPHICS_COLS = [ 'data.xmlResponse', 'data.name', diff --git a/utils/db_utils.py b/utils/db_utils.py index b2c898a..16565ce 100644 --- a/utils/db_utils.py +++ b/utils/db_utils.py @@ -118,17 +118,24 @@ def query_demographics(): return df -def query_trajectories(): +def query_trajectories(start_date, end_date): + start_ts, end_ts = None, datetime.max.timestamp() + if start_date is not None: + start_ts = datetime.combine(start_date, datetime.min.time()).timestamp() + + if end_date is not None: + end_ts = datetime.combine(end_date, datetime.max.time()).timestamp() ts = esta.TimeSeries.get_aggregate_time_series() entries = ts.find_entries( key_list=["analysis/recreated_location"], + time_query=estt.TimeQuery("data.ts", start_ts, end_ts), ) df = pd.json_normalize(list(entries)) if not df.empty: - for col in constants.BINARY_TRAJECTORIES_COLS: - if col in df.columns: - df[col] = df[col].apply(str) + for col in df.columns: + if df[col].dtype == 'object': + df[col] = df[col].apply(str) columns_to_drop = [col for col in df.columns if col.startswith("metadata")] df.drop(columns= columns_to_drop, inplace=True) modified_columns = perm_utils.get_trajectories_columns(df.columns)