Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix: Retain Current Page in UUID Table During Lazy Loading #142

Closed
wants to merge 5 commits into from
Closed

Fix: Retain Current Page in UUID Table During Lazy Loading #142

Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
dc80bc1
Page Loading finally stays
TeachMeTW Oct 10, 2024
d618664
Revered
TeachMeTW Oct 15, 2024
28e64a2
Update pages/data.py
TeachMeTW Oct 18, 2024
ac186e0
Update pages/data.py
TeachMeTW Oct 18, 2024
b3a23ca
Update pages/data.py
TeachMeTW Oct 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 42 additions & 36 deletions pages/data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,22 @@
layout = html.Div(
[
dcc.Markdown(intro),
dcc.Tabs(id="tabs-datatable", value='tab-uuids-datatable', children=[
dcc.Tab(label='UUIDs', value='tab-uuids-datatable'),
dcc.Tab(label='Trips', value='tab-trips-datatable'),
dcc.Tab(label='Demographics', value='tab-demographics-datatable'),
dcc.Tab(label='Trajectories', value='tab-trajectories-datatable'),
]),
dcc.Tabs(
id="tabs-datatable",
value='tab-uuids-datatable',
children=[
dcc.Tab(label='UUIDs', value='tab-uuids-datatable'),
dcc.Tab(label='Trips', value='tab-trips-datatable'),
dcc.Tab(label='Demographics', value='tab-demographics-datatable'),
dcc.Tab(label='Trajectories', value='tab-trajectories-datatable'),
]
),
Comment on lines +25 to +34
Copy link
Contributor

@JGreenlee JGreenlee Oct 18, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

formatting-only change, but acceptable

html.Div(id='tabs-content'),
dcc.Store(id='selected-tab', data='tab-uuids-datatable'), # Store to hold selected tab
dcc.Interval(id='interval-load-more', interval=20000, n_intervals=0), # default loading at 10s, can be lowered or hightened based on perf (usual process local is 3s)
dcc.Interval(id='interval-load-more', interval=24000, n_intervals=0), # Interval for loading more data
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Interval increased; ok for now until we implement a better solution that is not as fragile and does not depend on an arbitrary "magic number"

dcc.Store(id='store-uuids', data=[]), # Store to hold the original UUIDs data
dcc.Store(id='store-loaded-uuids', data={'data': [], 'loaded': False}), # Store to track loaded data
dcc.Store(id='uuids-page-current', data=0), # Store to track current page for UUIDs DataTable
# RadioItems for key list switch, wrapped in a div that can hide/show
html.Div(
id='keylist-switch-container',
Expand All @@ -54,7 +59,6 @@
)



def clean_location_data(df):
if 'data.start_loc.coordinates' in df.columns:
df['data.start_loc.coordinates'] = df['data.start_loc.coordinates'].apply(lambda x: f'({x[0]}, {x[1]})')
Expand Down Expand Up @@ -83,6 +87,17 @@ def show_keylist_switch(tab):
return {'display': 'none'} # Hide the keylist-switch on all other tabs


@callback(
Output('uuids-page-current', 'data'),
Input('uuid-table', 'page_current'),
State('tabs-datatable', 'value')
)
def update_uuids_page_current(page_current, selected_tab):
if selected_tab == 'tab-uuids-datatable':
return page_current
raise PreventUpdate


@callback(
Output('tabs-content', 'children'),
Output('store-loaded-uuids', 'data'),
Expand All @@ -98,69 +113,64 @@ def show_keylist_switch(tab):
Input('date-picker-timezone', 'value'),
Input('interval-load-more', 'n_intervals'), # Interval to trigger the loading of more data
Input('keylist-switch', 'value'), # Add keylist-switch to trigger data refresh on change
Input('uuids-page-current', 'data'), # Current page number for UUIDs DataTable
State('store-loaded-uuids', 'data'), # Use State to track already loaded data
State('store-loaded-uuids', 'loaded') # Keep track if we have finished loading all data
)
def render_content(tab, store_uuids, store_excluded_uuids, store_trips, store_demographics, store_trajectories,
start_date, end_date, timezone, n_intervals, key_list, loaded_uuids_store, all_data_loaded):
def render_content(
tab, store_uuids, store_excluded_uuids, store_trips, store_demographics, store_trajectories,
start_date, end_date, timezone, n_intervals, key_list, current_page,
loaded_uuids_store, all_data_loaded
):
Comment on lines +120 to +124
Copy link
Contributor

@JGreenlee JGreenlee Oct 18, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see; current_page added here so it can be used below

initial_batch_size = 10 # Define the batch size for loading UUIDs

# Update selected tab
selected_tab = tab
logging.debug(f"Selected tab: {selected_tab}")

# Handle the UUIDs tab without fullscreen loading spinner
if tab == 'tab-uuids-datatable':

TeachMeTW marked this conversation as resolved.
Show resolved Hide resolved
# Ensure store_uuids contains the key 'data' which is a list of dictionaries
if not isinstance(store_uuids, dict) or 'data' not in store_uuids:
logging.error(f"Expected store_uuids to be a dict with a 'data' key, but got {type(store_uuids)}")
return html.Div([html.P("Data structure error.")]), loaded_uuids_store, True

# Extract the list of UUIDs from the dict
uuids_list = store_uuids['data']

# Ensure uuids_list is a list for slicing
if not isinstance(uuids_list, list):
logging.error(f"Expected store_uuids['data'] to be a list but got {type(uuids_list)}")
return html.Div([html.P("Data structure error.")]), loaded_uuids_store, True

# Retrieve already loaded data from the store
loaded_data = loaded_uuids_store.get('data', [])
total_loaded = len(loaded_data)

# Handle lazy loading
if not loaded_uuids_store.get('loaded', False):
total_to_load = total_loaded + initial_batch_size
total_to_load = min(total_to_load, len(uuids_list)) # Avoid loading more than available

logging.debug(f"Loading next batch of UUIDs: {total_loaded} to {total_to_load}")

# Slice the list of UUIDs from the dict
new_data = uuids_list[total_loaded:total_to_load]

if new_data:
# Process and append the new data to the loaded store
processed_data = db_utils.add_user_stats(new_data, initial_batch_size)
loaded_data.extend(processed_data)

TeachMeTW marked this conversation as resolved.
Show resolved Hide resolved
# Update the store with the new data
loaded_uuids_store['data'] = loaded_data
loaded_uuids_store['loaded'] = len(loaded_data) >= len(uuids_list) # Mark all data as loaded if done

logging.debug(f"New batch loaded. Total loaded: {len(loaded_data)}")
loaded_uuids_store['data'] = loaded_data # Mark all data as loaded if done
loaded_uuids_store['loaded'] = len(loaded_data) >= len(uuids_list)

# Prepare the data to be displayed
columns = perm_utils.get_uuids_columns() # Get the relevant columns
df = pd.DataFrame(loaded_data)

if df.empty or not perm_utils.has_permission('data_uuids'):
logging.debug("No data or permission issues.")
return html.Div([html.P("No data available or you don't have permission.")]), loaded_uuids_store, True

df = df.drop(columns=[col for col in df.columns if col not in columns])

logging.debug("Returning appended data to update the UI.")
content = html.Div([
populate_datatable(df),
populate_datatable(df, table_id='uuid-table', page_current=current_page), # Pass current_page
TeachMeTW marked this conversation as resolved.
Show resolved Hide resolved
html.P(
f"Showing {len(loaded_data)} of {len(uuids_list)} UUIDs." +
(f" Loading 10 more..." if not loaded_uuids_store.get('loaded', False) else ""),
Expand All @@ -184,8 +194,8 @@ def render_content(tab, store_uuids, store_excluded_uuids, store_trips, store_de
df = df.drop(columns=[col for col in df.columns if col not in columns])
df = clean_location_data(df)

trips_table = populate_datatable(df, 'trips-table')
logging.debug(f"Returning 3 values: {trips_table}, {loaded_uuids_store}, True")
trips_table = populate_datatable(df)

return html.Div([
html.Button('Display columns with raw units', id='button-clicked', n_clicks=0, style={'marginLeft': '5px'}),
trips_table
Expand All @@ -210,9 +220,8 @@ def render_content(tab, store_uuids, store_excluded_uuids, store_trips, store_de

elif tab == 'tab-trajectories-datatable':
(start_date, end_date) = iso_to_date_only(start_date, end_date)

# Fetch new data based on the selected key_list from the keylist-switch
if store_trajectories == {} or key_list: # Ensure data is refreshed when key_list changes
if store_trajectories == {} or key_list:
store_trajectories = update_store_trajectories(start_date, end_date, timezone, store_excluded_uuids, key_list)

data = store_trajectories.get("data", [])
Expand All @@ -223,13 +232,9 @@ def render_content(tab, store_uuids, store_excluded_uuids, store_trips, store_de

df = pd.DataFrame(data)
if df.empty or not has_perm:
# If no permission or data, disable interval and return empty content
return None, loaded_uuids_store, True

# Filter the columns based on permissions
df = df.drop(columns=[col for col in df.columns if col not in columns])

# Return the populated DataTable
return populate_datatable(df), loaded_uuids_store, True

# Default case: if no data is loaded or the tab is not handled
Expand Down Expand Up @@ -277,19 +282,20 @@ def update_dropdowns_trips(n_clicks, button_label):
#return the list of hidden columns and the updated button label
return hidden_col, button_label

def populate_datatable(df, table_id=''):

def populate_datatable(df, table_id='', page_current=0):
if not isinstance(df, pd.DataFrame):
raise PreventUpdate
return dash_table.DataTable(
id= table_id,
id=table_id,
# columns=[{"name": i, "id": i} for i in df.columns],
data=df.to_dict('records'),
export_format="csv",
filter_options={"case": "sensitive"},
# filter_action="native",
sort_action="native", # give user capability to sort columns
sort_mode="single", # sort across 'multi' or 'single' columns
page_current=0, # page number that user is on
page_current=page_current, # set to current page
Comment on lines -292 to +296
Copy link
Contributor

@JGreenlee JGreenlee Oct 18, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note to self: this is basically the fix. Instead of this always being 0, we keep track of page_current and pass it in here

page_size=50, # number of rows visible per page
style_cell={
'textAlign': 'left',
Expand Down