diff --git a/src/dashboard/get_chart_data/get_chart_data.py b/src/dashboard/get_chart_data/get_chart_data.py index 38ec8b7..ab61fb1 100644 --- a/src/dashboard/get_chart_data/get_chart_data.py +++ b/src/dashboard/get_chart_data/get_chart_data.py @@ -29,7 +29,7 @@ "strContainsCI", "strStartsWithCI", "strEndsWithCI", - "strMatchesCI", + "matchesCI", "strNotEq", "strNotContains", "strNotStartsWith", @@ -89,7 +89,7 @@ "strContainsCI", "strStartsWithCI", "strEndsWithCI", - "strMatchesCI", + "matchesCI", "strNotEq", "strNotContains", "strNotStartsWith", @@ -165,9 +165,10 @@ def _build_query(query_params: dict, filter_groups: list, path_params: dict) -> raise errors.AggregatorFilterError( f"Invalid filter type {filter_config[1]} requested." ) - - inline_configs.append(config_params) - none_configs.append(none_params) + if config_params != []: + inline_configs.append(config_params) + if none_params != []: + none_configs.append(none_params) count_col = next(c for c in columns if c.startswith("cnt")) columns.remove(count_col) # these 'if in' checks is meant to handle the case where the selected column is also diff --git a/src/dashboard/get_chart_data/templates/filter_inline.sql.jinja b/src/dashboard/get_chart_data/templates/filter_inline.sql.jinja index 27d35b5..8383836 100644 --- a/src/dashboard/get_chart_data/templates/filter_inline.sql.jinja +++ b/src/dashboard/get_chart_data/templates/filter_inline.sql.jinja @@ -8,86 +8,86 @@ {#- TODO: replace all LIKE filters with regexp() calls -#} {#- Sting filters -#} {%- if filter_type == 'strEq' -%} -"{{ data }}" LIKE '{{ bound }}' +CAST("{{ data }}" AS VARCHAR) LIKE '{{ bound }}' {%- elif filter_type == 'strContains' -%} -"{{ data }}" LIKE '%{{ bound }}%' +CAST("{{ data }}" AS VARCHAR) LIKE '%{{ bound }}%' {%- elif filter_type == 'strStartsWith' -%} -"{{ data }}" LIKE '{{ bound }}%' +CAST("{{ data }}" AS VARCHAR) LIKE '{{ bound }}%' {%- elif filter_type == 'strEndsWith' -%} -"{{ data }}" LIKE '%{{ bound }}' +CAST("{{ data }}" AS VARCHAR) LIKE '%{{ bound }}' {%- elif filter_type == 'matches' -%} -regexp_like("{{ data }}", '{{ bound }}') +regexp_like(CAST("{{ data }}" AS VARCHAR), '{{ bound }}') {%- elif filter_type == 'strEqCI' -%} -"{{ data }}" ILIKE '{{ bound }}' +regexp_like(CAST("{{ data }}" AS VARCHAR), '(?i)^{{ bound }}$') {%- elif filter_type == 'strContainsCI' -%} -"{{ data }}" ILIKE '%{{ bound }}%' +regexp_like(CAST("{{ data }}" AS VARCHAR), '(?i){{ bound }}') {%- elif filter_type == 'strStartsWithCI' -%} -"{{ data }}" ILIKE '{{ bound }}%' +regexp_like(CAST("{{ data }}" AS VARCHAR), '(?i)^{{ bound }}') {%- elif filter_type == 'strEndsWithCI' -%} -"{{ data }}" ILIKE '%{{ bound }}' +regexp_like(CAST("{{ data }}" AS VARCHAR), '(?i){{ bound }}$') {%- elif filter_type == 'matchesCI' -%} -regexp_like("{{ data }}", '(?i){{ bound }}') +regexp_like(CAST("{{ data }}" AS VARCHAR), '(?i){{ bound }}') {%- elif filter_type == 'strNotEq' -%} -"{{ data }}" NOT LIKE '{{ bound }}' +CAST("{{ data }}" AS VARCHAR) NOT LIKE '{{ bound }}' {%- elif filter_type == 'strNotContains' -%} -"{{ data }}" NOT LIKE '%{{ bound }}%' +CAST("{{ data }}" AS VARCHAR) NOT LIKE '%{{ bound }}%' {%- elif filter_type == 'strNotStartsWith' -%} -"{{ data }}" NOT LIKE '{{ bound }}%' +CAST("{{ data }}" AS VARCHAR) NOT LIKE '{{ bound }}%' {%- elif filter_type == 'strNotEndsWith' -%} -"{{ data }}" NOT LIKE '%{{ bound }}' +CAST("{{ data }}" AS VARCHAR) NOT LIKE '%{{ bound }}' {%- elif filter_type == 'notMatches' -%} -NOT regexp_like("{{ data }}", '{{ bound }}') +NOT regexp_like(CAST("{{ data }}" AS VARCHAR), '{{ bound }}') {%- elif filter_type == 'strNotEqCI' -%} -"{{ data }}" NOT ILIKE '{{ bound }}' +NOT regexp_like(CAST("{{ data }}" AS VARCHAR), '(?i)^{{ bound }}$') {%- elif filter_type == 'strNotContainsCI' -%} -"{{ data }}" NOT ILIKE '%{{ bound }}%' +NOT regexp_like(CAST("{{ data }}" AS VARCHAR), '(?i){{ bound }}') {%- elif filter_type == 'strNotStartsWithCI' -%} -"{{ data }}" NOT ILIKE '{{ bound }}%' +NOT regexp_like(CAST("{{ data }}" AS VARCHAR), '(?i)^{{ bound }}') {%- elif filter_type == 'strNotEndsWithCI' -%} -"{{ data }}" NOT ILIKE '%{{ bound }}' +NOT regexp_like(CAST("{{ data }}" AS VARCHAR), '(?i){{ bound }}$') {%- elif filter_type == 'notMatchesCI' -%} -NOT regexp_like("{{ data }}", '(?i){{ bound }}') +NOT regexp_like(CAST("{{ data }}" AS VARCHAR), '(?i){{ bound }}') {#- Date filters -#} {%- elif filter_type == 'sameDay' -%} from_iso8601_timestamp("{{ data }}") = date_trunc('day',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'sameWeek' -%} -date_trunc('week',from_iso8601_timestamp('{{ data }}')) = date_trunc('week',from_iso8601_timestamp('{{ bound }}')) +date_trunc('week',from_iso8601_timestamp("{{ data }}")) = date_trunc('week',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'sameMonth' -%} -date_trunc('month',from_iso8601_timestamp('{{ data }}')) = date_trunc('month',from_iso8601_timestamp('{{ bound }}')) +date_trunc('month',from_iso8601_timestamp("{{ data }}")) = date_trunc('month',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'sameYear' -%} -date_trunc('year',from_iso8601_timestamp('{{ data }}')) = date_trunc('year',from_iso8601_timestamp('{{ bound }}')) +date_trunc('year',from_iso8601_timestamp("{{ data }}")) = date_trunc('year',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'sameDayOrBefore' -%} from_iso8601_timestamp("{{ data }}") <= date_trunc('day',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'sameWeekOrBefore' -%} -date_trunc('week',from_iso8601_timestamp('{{ data }}')) <= date_trunc('week',from_iso8601_timestamp('{{ bound }}')) +date_trunc('week',from_iso8601_timestamp("{{ data }}")) <= date_trunc('week',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'sameMonthOrBefore' -%} -date_trunc('month',from_iso8601_timestamp('{{ data }}')) <= date_trunc('month',from_iso8601_timestamp('{{ bound }}')) +date_trunc('month',from_iso8601_timestamp("{{ data }}")) <= date_trunc('month',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'sameYearOrBefore' -%} -date_trunc('year',from_iso8601_timestamp('{{ data }}')) <= date_trunc('year',from_iso8601_timestamp('{{ bound }}')) +date_trunc('year',from_iso8601_timestamp("{{ data }}")) <= date_trunc('year',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'sameDayOrAfter' -%} from_iso8601_timestamp("{{ data }}") >= date_trunc('day',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'sameWeekOrAfter' -%} -date_trunc('week',from_iso8601_timestamp('{{ data }}')) >= date_trunc('week',from_iso8601_timestamp('{{ bound }}')) +date_trunc('week',from_iso8601_timestamp("{{ data }}")) >= date_trunc('week',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'sameMonthOrAfter' -%} -date_trunc('month',from_iso8601_timestamp('{{ data }}')) >= date_trunc('month',from_iso8601_timestamp('{{ bound }}')) +date_trunc('month',from_iso8601_timestamp("{{ data }}")) >= date_trunc('month',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'sameYearOrAfter' -%} -date_trunc('year',from_iso8601_timestamp('{{ data }}')) >= date_trunc('year',from_iso8601_timestamp('{{ bound }}')) +date_trunc('year',from_iso8601_timestamp("{{ data }}")) >= date_trunc('year',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'beforeDay' -%} from_iso8601_timestamp("{{ data }}") < date_trunc('day',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'beforeWeek' -%} -date_trunc('week',from_iso8601_timestamp('{{ data }}')) < date_trunc('week',from_iso8601_timestamp('{{ bound }}')) +date_trunc('week',from_iso8601_timestamp("{{ data }}")) < date_trunc('week',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'beforeMonth' -%} -date_trunc('month',from_iso8601_timestamp('{{ data }}')) < date_trunc('month',from_iso8601_timestamp('{{ bound }}')) +date_trunc('month',from_iso8601_timestamp("{{ data }}")) < date_trunc('month',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'beforeYear' -%} -date_trunc('year',from_iso8601_timestamp('{{ data }}')) < date_trunc('year',from_iso8601_timestamp('{{ bound }}')) +date_trunc('year',from_iso8601_timestamp("{{ data }}")) < date_trunc('year',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'afterDay' -%} from_iso8601_timestamp("{{ data }}") > date_trunc('day',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'afterWeek' -%} -date_trunc('week',from_iso8601_timestamp('{{ data }}')) > date_trunc('week',from_iso8601_timestamp('{{ bound }}')) +date_trunc('week',from_iso8601_timestamp("{{ data }}")) > date_trunc('week',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'afterMonth' -%} -date_trunc('month',from_iso8601_timestamp('{{ data }}')) > date_trunc('month',from_iso8601_timestamp('{{ bound }}')) +date_trunc('month',from_iso8601_timestamp("{{ data }}")) > date_trunc('month',from_iso8601_timestamp('{{ bound }}')) {%- elif filter_type == 'afterYear' -%} -date_trunc('year',from_iso8601_timestamp('{{ data }}')) > date_trunc('year',from_iso8601_timestamp('{{ bound }}')) +date_trunc('year',from_iso8601_timestamp("{{ data }}")) > date_trunc('year',from_iso8601_timestamp('{{ bound }}')) {#- Boolean filters -#} {%- elif filter_type == 'isTrue' -%} "{{ data }}" IS TRUE diff --git a/src/dashboard/get_chart_data/templates/get_chart_data.sql.jinja b/src/dashboard/get_chart_data/templates/get_chart_data.sql.jinja index fc740ef..b10d96b 100644 --- a/src/dashboard/get_chart_data/templates/get_chart_data.sql.jinja +++ b/src/dashboard/get_chart_data/templates/get_chart_data.sql.jinja @@ -21,7 +21,7 @@ WHERE AND ( ( {{ data_column }} != 'cumulus__none' - {%- if inline_configs %} + {%- if inline_configs|length > 0 %} AND ( {{ inline.get_filters(inline_configs) }} @@ -30,7 +30,7 @@ WHERE ) OR ( {{ data_column }} = 'cumulus__none' - {%- if none_configs %} + {%- if none_configs|length > 0 %} AND ( {{ inline.get_filters(none_configs) }} diff --git a/tests/dashboard/test_filter_inline.py b/tests/dashboard/test_filter_inline.py index 2e01c94..014427a 100644 --- a/tests/dashboard/test_filter_inline.py +++ b/tests/dashboard/test_filter_inline.py @@ -12,140 +12,140 @@ ["col:strEq:str"], """ ( - "col" LIKE 'str' + CAST("col" AS VARCHAR) LIKE 'str' )""", ), ( ["col:strContains:str"], """ ( - "col" LIKE '%str%' + CAST("col" AS VARCHAR) LIKE '%str%' )""", ), ( ["col:strStartsWith:str"], """ ( - "col" LIKE 'str%' + CAST("col" AS VARCHAR) LIKE 'str%' )""", ), ( ["col:strEndsWith:str"], """ ( - "col" LIKE '%str' + CAST("col" AS VARCHAR) LIKE '%str' )""", ), ( ["col:matches:str"], """ ( - regexp_like("col", 'str') + regexp_like(CAST("col" AS VARCHAR), 'str') )""", ), ( ["col:strEqCI:str"], """ ( - "col" ILIKE 'str' + regexp_like(CAST("col" AS VARCHAR), '(?i)^str$') )""", ), ( ["col:strContainsCI:str"], """ ( - "col" ILIKE '%str%' + regexp_like(CAST("col" AS VARCHAR), '(?i)str') )""", ), ( ["col:strStartsWithCI:str"], """ ( - "col" ILIKE 'str%' + regexp_like(CAST("col" AS VARCHAR), '(?i)^str') )""", ), ( ["col:strEndsWithCI:str"], """ ( - "col" ILIKE '%str' + regexp_like(CAST("col" AS VARCHAR), '(?i)str$') )""", ), ( ["col:matchesCI:str"], """ ( - regexp_like("col", '(?i)str') + regexp_like(CAST("col" AS VARCHAR), '(?i)str') )""", ), ( ["col:strNotEq:str"], """ ( - "col" NOT LIKE 'str' + CAST("col" AS VARCHAR) NOT LIKE 'str' )""", ), ( ["col:strNotContains:str"], """ ( - "col" NOT LIKE '%str%' + CAST("col" AS VARCHAR) NOT LIKE '%str%' )""", ), ( ["col:strNotStartsWith:str"], """ ( - "col" NOT LIKE 'str%' + CAST("col" AS VARCHAR) NOT LIKE 'str%' )""", ), ( ["col:strNotEndsWith:str"], """ ( - "col" NOT LIKE '%str' + CAST("col" AS VARCHAR) NOT LIKE '%str' )""", ), ( ["col:notMatches:str"], """ ( - NOT regexp_like("col", 'str') + NOT regexp_like(CAST("col" AS VARCHAR), 'str') )""", ), ( ["col:strNotEqCI:str"], """ ( - "col" NOT ILIKE 'str' + NOT regexp_like(CAST("col" AS VARCHAR), '(?i)^str$') )""", ), ( ["col:strNotContainsCI:str"], """ ( - "col" NOT ILIKE '%str%' + NOT regexp_like(CAST("col" AS VARCHAR), '(?i)str') )""", ), ( ["col:strNotStartsWithCI:str"], """ ( - "col" NOT ILIKE 'str%' + NOT regexp_like(CAST("col" AS VARCHAR), '(?i)^str') )""", ), ( ["col:strNotEndsWithCI:str"], """ ( - "col" NOT ILIKE '%str' + NOT regexp_like(CAST("col" AS VARCHAR), '(?i)str$') )""", ), ( ["col:notMatchesCI:str"], """ ( - NOT regexp_like("col", '(?i)str') + NOT regexp_like(CAST("col" AS VARCHAR), '(?i)str') )""", ), ( @@ -232,13 +232,6 @@ "col" IS NOT NULL )""", ), - ( - ["col:strEq:str"], - """ - ( - "col" LIKE 'str' - )""", - ), ( ["column:sameDay:1900-01-01"], """ @@ -251,7 +244,7 @@ ["column:sameWeek:1900-01-01"], """ ( - date_trunc('week',from_iso8601_timestamp('column')) = """ + date_trunc('week',from_iso8601_timestamp("column")) = """ """date_trunc('week',from_iso8601_timestamp('1900-01-01')) )""", ), @@ -259,7 +252,7 @@ ["column:sameMonth:1900-01-01"], """ ( - date_trunc('month',from_iso8601_timestamp('column')) = """ + date_trunc('month',from_iso8601_timestamp("column")) = """ """date_trunc('month',from_iso8601_timestamp('1900-01-01')) )""", ), @@ -267,7 +260,7 @@ ["column:sameYear:1900-01-01"], """ ( - date_trunc('year',from_iso8601_timestamp('column')) = """ + date_trunc('year',from_iso8601_timestamp("column")) = """ """date_trunc('year',from_iso8601_timestamp('1900-01-01')) )""", ), @@ -283,7 +276,7 @@ ["column:sameWeekOrBefore:1900-01-01"], """ ( - date_trunc('week',from_iso8601_timestamp('column')) <= """ + date_trunc('week',from_iso8601_timestamp("column")) <= """ """date_trunc('week',from_iso8601_timestamp('1900-01-01')) )""", ), @@ -292,7 +285,7 @@ ( """ ( - date_trunc('month',from_iso8601_timestamp('column')) <= """ + date_trunc('month',from_iso8601_timestamp("column")) <= """ """date_trunc('month',from_iso8601_timestamp('1900-01-01')) )""" ), @@ -301,7 +294,7 @@ ["column:sameYearOrBefore:1900-01-01"], """ ( - date_trunc('year',from_iso8601_timestamp('column')) <= """ + date_trunc('year',from_iso8601_timestamp("column")) <= """ """date_trunc('year',from_iso8601_timestamp('1900-01-01')) )""", ), @@ -317,7 +310,7 @@ ["column:sameWeekOrAfter:1900-01-01"], """ ( - date_trunc('week',from_iso8601_timestamp('column')) >= """ + date_trunc('week',from_iso8601_timestamp("column")) >= """ """date_trunc('week',from_iso8601_timestamp('1900-01-01')) )""", ), @@ -326,7 +319,7 @@ ( """ ( - date_trunc('month',from_iso8601_timestamp('column')) >= """ + date_trunc('month',from_iso8601_timestamp("column")) >= """ """date_trunc('month',from_iso8601_timestamp('1900-01-01')) )""" ), @@ -335,7 +328,7 @@ ["column:sameYearOrAfter:1900-01-01"], """ ( - date_trunc('year',from_iso8601_timestamp('column')) >= """ + date_trunc('year',from_iso8601_timestamp("column")) >= """ """date_trunc('year',from_iso8601_timestamp('1900-01-01')) )""", ), @@ -351,7 +344,7 @@ ["column:beforeWeek:1900-01-01"], """ ( - date_trunc('week',from_iso8601_timestamp('column')) < """ + date_trunc('week',from_iso8601_timestamp("column")) < """ """date_trunc('week',from_iso8601_timestamp('1900-01-01')) )""", ), @@ -359,7 +352,7 @@ ["column:beforeMonth:1900-01-01"], """ ( - date_trunc('month',from_iso8601_timestamp('column')) < """ + date_trunc('month',from_iso8601_timestamp("column")) < """ """date_trunc('month',from_iso8601_timestamp('1900-01-01')) )""", ), @@ -367,7 +360,7 @@ ["column:beforeYear:1900-01-01"], """ ( - date_trunc('year',from_iso8601_timestamp('column')) < """ + date_trunc('year',from_iso8601_timestamp("column")) < """ """date_trunc('year',from_iso8601_timestamp('1900-01-01')) )""", ), @@ -383,7 +376,7 @@ ["column:afterWeek:1900-01-01"], """ ( - date_trunc('week',from_iso8601_timestamp('column')) > """ + date_trunc('week',from_iso8601_timestamp("column")) > """ """date_trunc('week',from_iso8601_timestamp('1900-01-01')) )""", ), @@ -391,7 +384,7 @@ ["column:afterMonth:1900-01-01"], """ ( - date_trunc('month',from_iso8601_timestamp('column')) > """ + date_trunc('month',from_iso8601_timestamp("column")) > """ """date_trunc('month',from_iso8601_timestamp('1900-01-01')) )""", ), @@ -399,7 +392,7 @@ ["column:afterYear:1900-01-01"], """ ( - date_trunc('year',from_iso8601_timestamp('column')) > """ + date_trunc('year',from_iso8601_timestamp("column")) > """ """date_trunc('year',from_iso8601_timestamp('1900-01-01')) )""", ), @@ -408,29 +401,29 @@ ["col:strEq:str", "col:strEqCI:str"], """ ( - "col" LIKE 'str' + CAST("col" AS VARCHAR) LIKE 'str' ) OR ( - "col" ILIKE 'str' + regexp_like(CAST("col" AS VARCHAR), '(?i)^str$') )""", ), ( ["col:strEq:str,col:strEqCI:str"], """ ( - "col" LIKE 'str' - AND "col" ILIKE 'str' + CAST("col" AS VARCHAR) LIKE 'str' + AND regexp_like(CAST("col" AS VARCHAR), '(?i)^str$') )""", ), ( ["col:strEq:str", "col:strEq:str,col:strEqCI:str"], """ ( - "col" LIKE 'str' + CAST("col" AS VARCHAR) LIKE 'str' ) OR ( - "col" LIKE 'str' - AND "col" ILIKE 'str' + CAST("col" AS VARCHAR) LIKE 'str' + AND regexp_like(CAST("col" AS VARCHAR), '(?i)^str$') )""", ), ], diff --git a/tests/dashboard/test_get_chart_data.py b/tests/dashboard/test_get_chart_data.py index fb52f12..33e9825 100644 --- a/tests/dashboard/test_get_chart_data.py +++ b/tests/dashboard/test_get_chart_data.py @@ -102,7 +102,7 @@ def mock_data_frame(filter_param): ( ( - "gender" LIKE 'female' + CAST("gender" AS VARCHAR) LIKE 'female' ) ) ) @@ -112,7 +112,7 @@ def mock_data_frame(filter_param): ( ( - "gender" LIKE 'female' + CAST("gender" AS VARCHAR) LIKE 'female' ) ) ) @@ -139,7 +139,7 @@ def mock_data_frame(filter_param): ( ( - "gender" LIKE 'cumulus__none' + CAST("gender" AS VARCHAR) LIKE 'cumulus__none' ) ) ) @@ -149,7 +149,7 @@ def mock_data_frame(filter_param): ( ( - "gender" LIKE 'cumulus__none' + CAST("gender" AS VARCHAR) LIKE 'cumulus__none' ) ) ) @@ -224,11 +224,11 @@ def mock_data_frame(filter_param): ( ( - regexp_like("gender", 'a') + regexp_like(CAST("gender" AS VARCHAR), 'a') ) OR ( - regexp_like("gender", 'e') - AND regexp_like("gender", 'm') + regexp_like(CAST("gender" AS VARCHAR), 'e') + AND regexp_like(CAST("gender" AS VARCHAR), 'm') ) ) ) @@ -238,11 +238,11 @@ def mock_data_frame(filter_param): ( ( - regexp_like("gender", 'a') + regexp_like(CAST("gender" AS VARCHAR), 'a') ) OR ( - regexp_like("gender", 'e') - AND regexp_like("gender", 'm') + regexp_like(CAST("gender" AS VARCHAR), 'e') + AND regexp_like(CAST("gender" AS VARCHAR), 'm') ) ) )