From 675011d2ded597d4718a9fdc92b187a16e267d0e Mon Sep 17 00:00:00 2001 From: Jijeong Lee Date: Sat, 4 May 2024 17:46:30 -0700 Subject: [PATCH 1/6] add getSurveyMetric endpoint --- emission/net/api/cfc_webapp.py | 83 ++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/emission/net/api/cfc_webapp.py b/emission/net/api/cfc_webapp.py index e585d6a25..b2002fcf1 100644 --- a/emission/net/api/cfc_webapp.py +++ b/emission/net/api/cfc_webapp.py @@ -306,6 +306,89 @@ def getUserProfile(): user = User.fromUUID(user_uuid) return user.getProfile() +@post('/get/metrics/survey') +def getSurveyMetric(): + logging.debug("Called getServeyMetric") + # todo : calculate survey metric + dummy_survey_metric = { + 'me': { + 'overview': { + 'answered': 5, + 'unanswered': 5, + 'mismatched': 0, + }, + 'rank' : 5, # 0-index + # for survey by trip cagetory + 'details': { + 'ev_roaming_trip': { + 'answered': 10, + 'unanswered': 5, + 'mismatched': 0, + }, + 'ev_return_trip': { + 'answered': 10, + 'unanswered': 10, + 'mismatched': 0, + }, + 'gas_car_trip': { + 'answered': 5, + 'unanswered': 10, + 'mismatched': 0, + }, + } + }, + 'others': { + 'overview': { + 'answered': 30, + 'unanswered': 60, + 'mismatched': 0, + }, + 'leaderboard': [ + { + 'answered': 10, + 'unanswered': 0, + 'mismatched': 0, + }, + { + 'answered': 9, + 'unanswered': 1, + 'mismatched': 0, + }, + { + 'answered': 8, + 'unanswered': 2, + 'mismatched': 0, + }, + { + 'answered': 7, + 'unanswered': 3, + 'mismatched': 0, + }, + { + 'answered': 6, + 'unanswered': 4, + 'mismatched': 0, + }, + { + 'answered': 4, + 'unanswered': 6, + 'mismatched': 0, + }, + { + 'answered': 2, + 'unanswered': 8, + 'mismatched': 0, + }, + { + 'answered': 1, + 'unanswered': 9, + 'mismatched': 0, + }, + ] + } + } + return dummy_survey_metric + @post('/result/metrics/') def summarize_metrics(time_type): _fill_aggregate_backward_compat(request) From d11d529e57443f8fe79a6335a98564864fd1fac6 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Tue, 21 May 2024 16:52:34 -0400 Subject: [PATCH 2/6] support 'yyyy_mm_dd' time type in 'result/metrics' endpoint The updated in-app dashboard will be using this new time type via `/result/metrics/yyyy_mm_dd`. When using this time type, `e-mission-common` is used for the metrics calculations, which has a new structure (https://github.com/e-mission/e-mission-server/pull/966#issuecomment-2119705314) `start_time` and `end_time` are given as ISO dates --- emission/net/api/cfc_webapp.py | 10 +++++++--- emission/net/api/metrics.py | 19 +++++++++++++++++-- .../storage/decorations/local_date_queries.py | 12 ++++++++++++ setup/environment36.yml | 2 +- 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/emission/net/api/cfc_webapp.py b/emission/net/api/cfc_webapp.py index b2002fcf1..82ea12b15 100644 --- a/emission/net/api/cfc_webapp.py +++ b/emission/net/api/cfc_webapp.py @@ -411,14 +411,18 @@ def summarize_metrics(time_type): else: old_style = True is_return_aggregate = True + + app_config = request.json['app_config'] if 'app_config' in request.json else None + time_type_map = { - 'timestamp': metrics.summarize_by_timestamp, - 'local_date': metrics.summarize_by_local_date + 'timestamp': metrics.summarize_by_timestamp, # used by old UI + 'local_date': metrics.summarize_by_local_date, + 'yyyy_mm_dd': metrics.summarize_by_yyyy_mm_dd # used by new UI } metric_fn = time_type_map[time_type] ret_val = metric_fn(user_uuid, start_time, end_time, - freq_name, metric_list, is_return_aggregate) + freq_name, metric_list, is_return_aggregate, app_config) if old_style: logging.debug("old_style metrics found, returning array of entries instead of array of arrays") assert(len(metric_list) == 1) diff --git a/emission/net/api/metrics.py b/emission/net/api/metrics.py index 9bd5ffdaa..a427d9c31 100644 --- a/emission/net/api/metrics.py +++ b/emission/net/api/metrics.py @@ -10,16 +10,31 @@ import emission.analysis.result.metrics.time_grouping as earmt import emission.analysis.result.metrics.simple_metrics as earms +import emission.storage.decorations.analysis_timeseries_queries as esda +import emission.storage.decorations.local_date_queries as esdl +import emission.storage.timeseries.tcquery as esttc -def summarize_by_timestamp(user_id, start_ts, end_ts, freq, metric_list, include_aggregate): +import emcommon.metrics.metrics_summaries as emcms + +def summarize_by_timestamp(user_id, start_ts, end_ts, freq, metric_list, include_aggregate, app_config=None): return _call_group_fn(earmt.group_by_timestamp, user_id, start_ts, end_ts, freq, metric_list, include_aggregate) -def summarize_by_local_date(user_id, start_ld, end_ld, freq_name, metric_list, include_aggregate): +def summarize_by_local_date(user_id, start_ld, end_ld, freq_name, metric_list, include_aggregate, app_config=None): local_freq = earmt.LocalFreq[freq_name] return _call_group_fn(earmt.group_by_local_date, user_id, start_ld, end_ld, local_freq, metric_list, include_aggregate) +def summarize_by_yyyy_mm_dd(user_id, start_ymd, end_ymd, freq, metric_list, include_agg, app_config): + time_query = esttc.TimeComponentQuery( + "data.start_local_dt", + esdl.yyyy_mm_dd_to_local_date(start_ymd), + esdl.yyyy_mm_dd_to_local_date(end_ymd) + ) + trips = esda.get_entries(esda.COMPOSITE_TRIP_KEY, None, time_query) + print('found ' + str([e for e in trips])) + return emcms.generate_summaries(metric_list, trips, app_config) + def _call_group_fn(group_fn, user_id, start_time, end_time, freq, metric_list, include_aggregate): summary_fn_list = [earms.get_summary_fn(metric_name) for metric_name in metric_list] diff --git a/emission/storage/decorations/local_date_queries.py b/emission/storage/decorations/local_date_queries.py index 8a0e2d149..d6aca9fbb 100644 --- a/emission/storage/decorations/local_date_queries.py +++ b/emission/storage/decorations/local_date_queries.py @@ -48,3 +48,15 @@ def get_comparison_query(field_prefix, base_ld, limit_ld, units, gt_or_lt): return { "$or": or_conditions } else: return {} + +def yyyy_mm_dd_to_local_date(ymd: str) -> ecwl.LocalDate: + return ecwl.LocalDate({ + 'year': int(ymd[0:4]), + 'month': int(ymd[5:7]), + 'day': int(ymd[8:10]) + }) + +def get_yyyy_mm_dd_range_query(field_name, start_ymd: str, end_ymd: str) -> dict: + start_local_date = ymd_to_local_date(start_ymd) + end_local_date = ymd_to_local_date(end_ymd) + return get_range_query(field_name, start_local_date, end_local_date) diff --git a/setup/environment36.yml b/setup/environment36.yml index d02231754..49dc9c1fe 100644 --- a/setup/environment36.yml +++ b/setup/environment36.yml @@ -19,7 +19,7 @@ dependencies: - scipy=1.10.0 - utm=0.7.0 - pip: - - git+https://github.com/JGreenlee/e-mission-common@0.4.3 + - git+https://github.com/JGreenlee/e-mission-common@0.5.0 - pyfcm==1.5.4 - pygeocoder==1.2.5 - pymongo==4.3.3 From 05d78f3eec3dd6508d404ca70478eced21fbc903 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Tue, 21 May 2024 16:53:56 -0400 Subject: [PATCH 3/6] remove dummy survey data --- emission/net/api/cfc_webapp.py | 83 ---------------------------------- 1 file changed, 83 deletions(-) diff --git a/emission/net/api/cfc_webapp.py b/emission/net/api/cfc_webapp.py index 82ea12b15..a5e0d1dd9 100644 --- a/emission/net/api/cfc_webapp.py +++ b/emission/net/api/cfc_webapp.py @@ -306,89 +306,6 @@ def getUserProfile(): user = User.fromUUID(user_uuid) return user.getProfile() -@post('/get/metrics/survey') -def getSurveyMetric(): - logging.debug("Called getServeyMetric") - # todo : calculate survey metric - dummy_survey_metric = { - 'me': { - 'overview': { - 'answered': 5, - 'unanswered': 5, - 'mismatched': 0, - }, - 'rank' : 5, # 0-index - # for survey by trip cagetory - 'details': { - 'ev_roaming_trip': { - 'answered': 10, - 'unanswered': 5, - 'mismatched': 0, - }, - 'ev_return_trip': { - 'answered': 10, - 'unanswered': 10, - 'mismatched': 0, - }, - 'gas_car_trip': { - 'answered': 5, - 'unanswered': 10, - 'mismatched': 0, - }, - } - }, - 'others': { - 'overview': { - 'answered': 30, - 'unanswered': 60, - 'mismatched': 0, - }, - 'leaderboard': [ - { - 'answered': 10, - 'unanswered': 0, - 'mismatched': 0, - }, - { - 'answered': 9, - 'unanswered': 1, - 'mismatched': 0, - }, - { - 'answered': 8, - 'unanswered': 2, - 'mismatched': 0, - }, - { - 'answered': 7, - 'unanswered': 3, - 'mismatched': 0, - }, - { - 'answered': 6, - 'unanswered': 4, - 'mismatched': 0, - }, - { - 'answered': 4, - 'unanswered': 6, - 'mismatched': 0, - }, - { - 'answered': 2, - 'unanswered': 8, - 'mismatched': 0, - }, - { - 'answered': 1, - 'unanswered': 9, - 'mismatched': 0, - }, - ] - } - } - return dummy_survey_metric - @post('/result/metrics/') def summarize_metrics(time_type): _fill_aggregate_backward_compat(request) From 4d87bd7695d35a49bfb1e674476366b1039d4280 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Fri, 24 May 2024 11:23:46 -0400 Subject: [PATCH 4/6] use e-mission-common @ 0.5.1 includes bugfixes for metrics calculations https://github.com/JGreenlee/e-mission-common/compare/0.5.0...0.5.1 --- setup/environment36.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/environment36.yml b/setup/environment36.yml index 49dc9c1fe..b886ca984 100644 --- a/setup/environment36.yml +++ b/setup/environment36.yml @@ -19,7 +19,7 @@ dependencies: - scipy=1.10.0 - utm=0.7.0 - pip: - - git+https://github.com/JGreenlee/e-mission-common@0.5.0 + - git+https://github.com/JGreenlee/e-mission-common@0.5.1 - pyfcm==1.5.4 - pygeocoder==1.2.5 - pymongo==4.3.3 From 1f0043409298394dc4f2ebdce4d9b04caa822eb1 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 29 May 2024 00:27:23 -0400 Subject: [PATCH 5/6] fix ymd -> yyyy_mm_dd typo Co-authored-by: K. Shankari --- emission/storage/decorations/local_date_queries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emission/storage/decorations/local_date_queries.py b/emission/storage/decorations/local_date_queries.py index d6aca9fbb..ccd1bf26e 100644 --- a/emission/storage/decorations/local_date_queries.py +++ b/emission/storage/decorations/local_date_queries.py @@ -57,6 +57,6 @@ def yyyy_mm_dd_to_local_date(ymd: str) -> ecwl.LocalDate: }) def get_yyyy_mm_dd_range_query(field_name, start_ymd: str, end_ymd: str) -> dict: - start_local_date = ymd_to_local_date(start_ymd) + start_local_date = yyyy_mm_dd_to_local_date(start_ymd) end_local_date = ymd_to_local_date(end_ymd) return get_range_query(field_name, start_local_date, end_local_date) From 5c20021a44efeeac6d54fa1246d3f1b0ecefc5ca Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 29 May 2024 00:30:38 -0400 Subject: [PATCH 6/6] fix ymd -> yyyy_mm_dd typo --- emission/storage/decorations/local_date_queries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emission/storage/decorations/local_date_queries.py b/emission/storage/decorations/local_date_queries.py index ccd1bf26e..bc6e8bc53 100644 --- a/emission/storage/decorations/local_date_queries.py +++ b/emission/storage/decorations/local_date_queries.py @@ -58,5 +58,5 @@ def yyyy_mm_dd_to_local_date(ymd: str) -> ecwl.LocalDate: def get_yyyy_mm_dd_range_query(field_name, start_ymd: str, end_ymd: str) -> dict: start_local_date = yyyy_mm_dd_to_local_date(start_ymd) - end_local_date = ymd_to_local_date(end_ymd) + end_local_date = yyyy_mm_dd_to_local_date(end_ymd) return get_range_query(field_name, start_local_date, end_local_date)