From 52cf6440fff231234ba531630a2574679c5ce504 Mon Sep 17 00:00:00 2001 From: cedricvlt Date: Sat, 30 Sep 2023 18:41:39 +0200 Subject: [PATCH] v0.1.2 - Add DataFrame support --- README.md | 59 +++-- setup.py | 2 +- streamlit_condition_tree/__init__.py | 56 +++++ streamlit_condition_tree/example.py | 40 ++- .../frontend/package.json | 2 +- .../frontend/src/ConditionTree.tsx | 230 +++++++++--------- .../frontend/src/config.ts | 134 ++++++++++ 7 files changed, 372 insertions(+), 151 deletions(-) create mode 100644 streamlit_condition_tree/frontend/src/config.ts diff --git a/README.md b/README.md index fc78979..7d7b6b7 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Based on [react-awesome-query-builder](https://github.com/ukrbublik/react-awesom Check out [live demo](https://condition-tree-demo.streamlit.app/) ! -This component allows users to build complex condition trees that can be used, for example, to filter a dataframe or build a query. +This component allows users to build complex condition trees that can be used to filter a dataframe or build a query. preview @@ -33,11 +33,37 @@ This component allows users to build complex condition trees that can be used, f ## Basic usage +### Filter a dataframe + +```python +import pandas as pd +from streamlit_condition_tree import condition_tree, config_from_dataframe + +# Initial dataframe +df = pd.DataFrame({ + 'First Name': ['Georges', 'Alfred'], + 'Age': [45, 98], + 'Favorite Color': ['Green', 'Red'], + 'Like Tomatoes': [True, False] +}) + +# Basic field configuration from dataframe +config = config_from_dataframe(df) + +# Condition tree +query_string = condition_tree(config) + +# Filtered dataframe +df = df.query(query_string) +``` + +### Build a query + ```python import streamlit as st from streamlit_condition_tree import condition_tree - +# Build a custom configuration config = { 'fields': { 'name': { @@ -58,11 +84,13 @@ config = { } } +# Condition tree return_val = condition_tree( config, return_type='sql' ) +# Generated SQL st.write(return_val) ``` @@ -72,18 +100,22 @@ st.write(return_val) ```python def condition_tree( - config: Dict - return_type: str - tree: Dict - min_height: int - placeholder: str + config: dict, + return_type: str, + tree: dict, + min_height: int, + placeholder: str, key: str ) ``` -- **config**: Python dictionary that resembles the JSON counterpart of - the React component [config](https://github.com/ukrbublik/react-awesome-query-builder/blob/master/CONFIG.adoc). -*Note*: Javascript functions (ex: validators) are not yet supported. +- **config**: Python dictionary (mostly used to define the fields) that resembles the JSON counterpart of + the React component. + +A basic configuration can be built from a DataFrame with `config_from_dataframe`. +For a more advanced configuration, see the component [doc](https://github.com/ukrbublik/react-awesome-query-builder/blob/master/CONFIG.adoc) +and [demo](https://ukrbublik.github.io/react-awesome-query-builder/). + *Note*: Javascript functions (ex: validators) are not yet supported. - **return_type**: Format of the returned value : @@ -92,9 +124,9 @@ def condition_tree( - sql - spel - elasticSearch - - jsonLogic - - Default : queryString + - jsonLogic + + Default : queryString (can be used to filter a pandas DataFrame using DataFrame.query) - **tree**: Input condition tree (see section below) @@ -125,5 +157,4 @@ It can be loaded as an input tree through the `tree` parameter. ## Potential future improvements -- **Dataframe filtering support**: automatically build config from dataframe and return a query string adapted to `pandas.DataFrame.query` - **Javascript support**: allow injection of javascript code in the configuration (e.g. validators) diff --git a/setup.py b/setup.py index 1b33872..18aa094 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setuptools.setup( name="streamlit-condition-tree", - version="0.1.1", + version="0.1.2", author="Cédric Villette", author_email="cedric_villette@hotmail.fr", description="Condition Tree Builder for Streamlit", diff --git a/streamlit_condition_tree/__init__.py b/streamlit_condition_tree/__init__.py index 95a07c5..6e4ec1d 100644 --- a/streamlit_condition_tree/__init__.py +++ b/streamlit_condition_tree/__init__.py @@ -15,6 +15,43 @@ _component_func = components.declare_component("streamlit_condition_tree", path=build_dir) +type_mapper = { + 'b': 'boolean', + 'i': 'number', + 'u': 'number', + 'f': 'number', + 'c': '', + 'm': '', + 'M': 'datetime', + 'O': 'text', + 'S': 'text', + 'U': 'text', + 'V': '' +} + + +def config_from_dataframe(dataframe): + """Return a basic configuration from dataframe columns""" + + fields = {} + for col_name, col_dtype in zip(dataframe.columns, dataframe.dtypes): + col_type = 'select' if col_dtype == 'category' else type_mapper[col_dtype.kind] + + if col_type: + col_config = { + 'label': col_name, + 'type': col_type + } + if col_type == 'select': + categories = dataframe[col_name].cat.categories + col_config['fieldSettings'] = { + 'listValues': [{'value': c, 'title': c} for c in categories] + } + fields[f'{col_name}'] = col_config + + return {'fields': fields} + + def condition_tree(config: dict, return_type: str = 'queryString', tree: dict = None, @@ -32,12 +69,16 @@ def condition_tree(config: dict, Format in which output should be returned to streamlit. Possible values : queryString | mongodb | sql | spel | elasticSearch | jsonLogic. + Default : queryString (compatible with DataFrame.query) tree: dict or None Input condition tree + Default: None min_height: int Minimum height of the component frame + Default: 400 placeholder: str Text displayed when the condition tree is empty + Default: empty key: str or None An optional key that uniquely identifies this component. If this is None, and the component's arguments are changed, the component will @@ -51,6 +92,16 @@ def condition_tree(config: dict, """ + if return_type == 'queryString': + # Add backticks to fields with space in their name + fields = {} + for field_name, field_config in config['fields'].items(): + if ' ' in field_name: + field_name = f'`{field_name}`' + fields[field_name] = field_config + + config['fields'] = fields + output_tree, component_value = _component_func( config=config, return_type=return_type, @@ -60,6 +111,11 @@ def condition_tree(config: dict, placeholder=placeholder, default=['', ''] ) + + if return_type == 'queryString' and not component_value: + # Default string that returns all the values in DataFrame.query + component_value = 'index in index' + st.session_state[key] = output_tree return component_value diff --git a/streamlit_condition_tree/example.py b/streamlit_condition_tree/example.py index bb3af32..f2ed93b 100644 --- a/streamlit_condition_tree/example.py +++ b/streamlit_condition_tree/example.py @@ -1,30 +1,26 @@ +import numpy as np +import pandas as pd import streamlit as st -from streamlit_condition_tree import condition_tree +from streamlit_condition_tree import condition_tree, config_from_dataframe +df = pd.read_csv( + 'https://media.githubusercontent.com/media/datablist/sample-csv-files/main/files/people/people-100.csv', + index_col=0, + parse_dates=['Date of birth'], + date_format='%Y-%m-%d') +df['Age'] = ((pd.Timestamp.today() - df['Date of birth']).dt.days / 365).astype(int) +df['Sex'] = pd.Categorical(df['Sex']) +df['Likes tomatoes'] = np.random.randint(2, size=df.shape[0]).astype(bool) -config = { - 'fields': { - 'name': { - 'label': 'Name', - 'type': 'text', - }, - 'qty': { - 'label': 'Age', - 'type': 'number', - 'fieldSettings': { - 'min': 0 - }, - }, - 'like_tomatoes': { - 'label': 'Likes tomatoes', - 'type': 'boolean', - } - } -} +st.dataframe(df) + +config = config_from_dataframe(df) return_val = condition_tree( config, - return_type='sql' ) -st.write(return_val) +st.code(return_val) + +df = df.query(return_val) +st.dataframe(df) diff --git a/streamlit_condition_tree/frontend/package.json b/streamlit_condition_tree/frontend/package.json index 9c025d7..1fa81dc 100644 --- a/streamlit_condition_tree/frontend/package.json +++ b/streamlit_condition_tree/frontend/package.json @@ -1,6 +1,6 @@ { "name": "streamlit-condition-tree", - "version": "0.1.1", + "version": "0.1.2", "private": true, "dependencies": { "@fontsource/source-sans-pro": "^5.0.8", diff --git a/streamlit_condition_tree/frontend/src/ConditionTree.tsx b/streamlit_condition_tree/frontend/src/ConditionTree.tsx index 1327777..43cd71c 100644 --- a/streamlit_condition_tree/frontend/src/ConditionTree.tsx +++ b/streamlit_condition_tree/frontend/src/ConditionTree.tsx @@ -7,140 +7,144 @@ import {ConfigProvider, theme as antdTheme} from 'antd'; import '@react-awesome-query-builder/antd/css/styles.css'; import './style.css' import "@fontsource/source-sans-pro"; +import {defaultConfig} from './config' interface State { - tree: ImmutableTree, - config: Config + tree: ImmutableTree, + config: Config } -const defaultTree : JsonGroup = { - type: "group", - id: QbUtils.uuid() +const defaultTree: JsonGroup = { + type: "group", + id: QbUtils.uuid() }; const exportFunctions: Record = { - queryString: QbUtils.queryString, - mongodb: QbUtils.mongodbFormat, - sql: QbUtils.sqlFormat, - spel: QbUtils.spelFormat, - elasticSearch: QbUtils.elasticSearchFormat, - jsonLogic: QbUtils.jsonLogicFormat + queryString: QbUtils.queryString, + mongodb: QbUtils.mongodbFormat, + sql: QbUtils.sqlFormat, + spel: QbUtils.spelFormat, + elasticSearch: QbUtils.elasticSearchFormat, + jsonLogic: QbUtils.jsonLogicFormat } const formatTree = (tree: any) => { - // Recursively add uuid and rename 'children' key - tree.id = QbUtils.uuid() - if (tree.children) { - tree.children1 = tree.children; - delete tree.children; - tree.children1.forEach(formatTree); - } + // Recursively add uuid and rename 'children' key + tree.id = QbUtils.uuid() + if (tree.children) { + tree.children1 = tree.children; + delete tree.children; + tree.children1.forEach(formatTree); + } }; const unformatTree = (tree: any) => { - // Recursively remove uuid and rename 'children1' key - delete tree.id; - if (tree.children1) { - tree.children = tree.children1; - delete tree.children1; - tree.children.forEach(unformatTree); - } + // Recursively remove uuid and rename 'children1' key + delete tree.id; + if (tree.children1) { + tree.children = tree.children1; + delete tree.children1; + tree.children.forEach(unformatTree); + } }; class ConditionTree extends StreamlitComponentBase { - public constructor(props: ComponentProps) { - super(props); - - const config: Config = { - ...(AntdConfig), - ...props.args['config'] - }; - - // Load input tree - let tree : ImmutableTree = QbUtils.loadTree(defaultTree) - if (props.args['tree'] != null) { - try { - let input_tree = props.args['tree'] - formatTree(input_tree) - tree = QbUtils.checkTree(QbUtils.loadTree(input_tree), config) - } catch (error) { - console.log(error); - } + public constructor(props: ComponentProps) { + super(props); + + console.log(AntdConfig) + + const config: Config = { + ...defaultConfig, + ...props.args['config'] + }; + + // Load input tree + let tree: ImmutableTree = QbUtils.loadTree(defaultTree) + if (props.args['tree'] != null) { + try { + let input_tree = props.args['tree'] + formatTree(input_tree) + tree = QbUtils.checkTree(QbUtils.loadTree(input_tree), config) + } catch (error) { + console.log(error); + } + } + + this.state = {config, tree} + this.setStreamlitValue(tree) + } + + public render = (): ReactNode => { + const {theme} = this.props + const tree = QbUtils.getTree(this.state.tree) + const empty = !tree.children1 || !tree.children1.length + + return ( +
+ + +

{empty && this.props.args['placeholder']}

+ +
+ ) + } + + componentDidUpdate = () => { + // Class to apply custom css on rule_groups with a single child + document + .querySelectorAll('.rule_group>.group--children:has(> :nth-child(1):last-child)') + .forEach((x) => x.classList.add('single-child')) + + // Set frame height + const height = Math.max( + document.body.scrollHeight + 20, + this.props.args['min_height'] + ); + Streamlit.setFrameHeight(height); } - this.state = { config, tree } - this.setStreamlitValue(tree) - } - - public render = (): ReactNode => { - const {theme} = this.props - const tree = QbUtils.getTree(this.state.tree) - const empty = !tree.children1 || !tree.children1.length - - return( -
- - -

{empty && this.props.args['placeholder']}

- -
+ private onChange = (immutableTree: ImmutableTree) => { + this.setState({tree: immutableTree}) + this.setStreamlitValue(immutableTree) + } + + private setStreamlitValue = (tree: ImmutableTree) => { + const exportFunc = exportFunctions[this.props.args['return_type']] + const exportValue = exportFunc ? exportFunc(tree, this.state.config) : '' + + let output_tree: JsonTree = QbUtils.getTree(tree) + unformatTree(output_tree) + Streamlit.setComponentValue([output_tree, exportValue]) + } + + private renderBuilder = (props: BuilderProps) => ( +
+
+ +
+
) - } - - private onChange = (immutableTree: ImmutableTree) => { - this.setState({ tree: immutableTree }) - this.setStreamlitValue(immutableTree) - } - - private setStreamlitValue = (tree: ImmutableTree) => { - const exportFunc = exportFunctions[this.props.args['return_type']] - const exportValue = exportFunc ? exportFunc(tree, this.state.config) : '' - - let output_tree : JsonTree = QbUtils.getTree(tree) - unformatTree(output_tree) - Streamlit.setComponentValue([output_tree, exportValue]) - } - - private renderBuilder = (props: BuilderProps) => ( -
-
- -
-
- ) - - componentDidUpdate = () => { - // Class to apply custom css on rule_groups with a single child - document - .querySelectorAll('.rule_group>.group--children:has(> :nth-child(1):last-child)') - .forEach((x)=> x.classList.add('single-child')) - - // Set frame height - const height = Math.max( - document.body.scrollHeight+20, - this.props.args['min_height'] - ); - Streamlit.setFrameHeight(height); - } } + export default withStreamlitConnection(ConditionTree); diff --git a/streamlit_condition_tree/frontend/src/config.ts b/streamlit_condition_tree/frontend/src/config.ts new file mode 100644 index 0000000..f48af52 --- /dev/null +++ b/streamlit_condition_tree/frontend/src/config.ts @@ -0,0 +1,134 @@ +import {AntdConfig, Config} from "@react-awesome-query-builder/antd"; + +const defaultConfig: Config = { + ...(AntdConfig), + conjunctions: { + AND: { + ...AntdConfig.conjunctions.AND, + formatConj: (children, _conj, not) => + (not ? '~' : '') + '(' + children.join(' & ') + ')', + }, + OR: { + ...AntdConfig.conjunctions.OR, + formatConj: (children, _conj, not) => + (not ? '~' : '') + '(' + children.join(' | ') + ')', + } + }, + operators: { + ...AntdConfig.operators, + equal: { + ...AntdConfig.operators.equal, + formatOp: (field, op, value, valueSrcs, valueTypes, opDef, operatorOptions, isForDisplay, fieldDef) => { + if (valueTypes == "boolean") { + return value == 'true' ? field : `~${field}` + } + return `${field} == ${value}`; + } + }, + not_equal: { + ...AntdConfig.operators.not_equal, + formatOp: (field, op, value, valueSrcs, valueTypes, opDef, operatorOptions, isForDisplay, fieldDef) => { + if (valueTypes == "boolean") { + return value == 'true' ? `~${field}` : field + } + return `${field} != ${value}`; + } + }, + between: { + ...AntdConfig.operators.between, + formatOp: (field, op, values, valueSrcs, valueTypes, opDef, operatorOptions, isForDisplay) => + typeof values !== 'string' ? `(${values.first()} <= ${field} <= ${values.get(1)})` : '' + }, + not_between: { + ...AntdConfig.operators.not_between, + formatOp: (field, op, values, valueSrcs, valueTypes, opDef, operatorOptions, isForDisplay) => + typeof values !== 'string' ? `~(${values.first()} <= ${field} <= ${values.get(1)})` : '' + }, + is_null: { + ...AntdConfig.operators.is_null, + formatOp: (field, op, value, valueSrc, valueType, opDef, operatorOptions, isForDisplay) => + `${field}.isnull()` + }, + is_not_null: { + ...AntdConfig.operators.is_not_null, + formatOp: (field, op, value, valueSrc, valueType, opDef, operatorOptions, isForDisplay) => + `~${field}.isnull()` + }, + like: { + ...AntdConfig.operators.like, + formatOp: (field, op, value, valueSrc, valueType, opDef, operatorOptions, isForDisplay) => + `${field}.str.contains(${value}, regex=True)` + }, + not_like: { + ...AntdConfig.operators.not_like, + formatOp: (field, op, value, valueSrc, valueType, opDef, operatorOptions, isForDisplay) => + `~${field}.str.contains(${value}, regex=True)` + }, + starts_with: { + ...AntdConfig.operators.starts_with, + formatOp: (field, op, value, valueSrc, valueType, opDef, operatorOptions, isForDisplay) => + `${field}.str.startswith(${value})` + }, + ends_with: { + ...AntdConfig.operators.ends_with, + formatOp: (field, op, value, valueSrc, valueType, opDef, operatorOptions, isForDisplay) => + `${field}.str.endswith(${value})` + }, + is_empty: { + ...AntdConfig.operators.is_empty, + formatOp: (field, op, value, valueSrc, valueType, opDef, operatorOptions, isForDisplay) => + `${field} == ""` + }, + is_not_empty: { + ...AntdConfig.operators.is_not_empty, + formatOp: (field, op, value, valueSrc, valueType, opDef, operatorOptions, isForDisplay) => + `${field} != ""` + }, + select_any_in: { + ...AntdConfig.operators.select_any_in, + formatOp: (field, op, values, valueSrc, valueType, opDef, operatorOptions, isForDisplay) => { + if (valueSrc == "value" && Array.isArray(values)) + return `${field}.isin([${values.join(", ")}])`; + else + return `${field}.isin(${values})`; + }, + }, + select_not_any_in: { + ...AntdConfig.operators.select_not_any_in, + formatOp: (field, op, values, valueSrc, valueType, opDef, operatorOptions, isForDisplay) => { + if (valueSrc == "value" && Array.isArray(values)) + return `~${field}.isin([${values.join(", ")}])`; + else + return `~${field}.isin(${values})`; + }, + }, + multiselect_contains: { + ...AntdConfig.operators.multiselect_contains, + formatOp: (field, op, values, valueSrc, valueType, opDef, operatorOptions, isForDisplay) => { + if (valueSrc == "value" && Array.isArray(values)) { + return `${field}.str.contains([${values.join(", ")}])`; + } else + return `${field}.str.contains(${values})`; + }, + }, + multiselect_not_contains: { + ...AntdConfig.operators.multiselect_not_contains, + formatOp: (field, op, values, valueSrc, valueType, opDef, operatorOptions, isForDisplay) => { + if (valueSrc == "value" && Array.isArray(values)) { + return `~${field}.str.contains([${values.join(", ")}])`; + } else + return `~${field}.str.contains(${values})`; + }, + } + + }, + types: { + ...AntdConfig.types, + text: { + ...AntdConfig.types.text, + excludeOperators: ["proximity"], + } + } +} + +export {defaultConfig} \ No newline at end of file