From 26ec85348066e9e30e3c96e4a7f4457dbec59666 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Adriano?= Date: Mon, 1 Apr 2024 14:41:41 +0200 Subject: [PATCH 1/5] Add simple cli interface --- jaiqu/cli.py | 59 +++++++++++++++++++++++++++++++++++++++++++++ jaiqu/jaiqu.py | 12 ++++----- pyproject.toml | 6 ++++- samples/data.json | 11 +++++++++ samples/schema.json | 22 +++++++++++++++++ 5 files changed, 103 insertions(+), 7 deletions(-) create mode 100644 jaiqu/cli.py create mode 100644 samples/data.json create mode 100644 samples/schema.json diff --git a/jaiqu/cli.py b/jaiqu/cli.py new file mode 100644 index 0000000..11cf2ec --- /dev/null +++ b/jaiqu/cli.py @@ -0,0 +1,59 @@ +import json +import sys + +from typer import Option, Typer + +from .jaiqu import translate_schema + +typer_app = Typer() + + +@typer_app.command() +def jaiqu( + schema_file: str = Option(..., "-s", "--schema", help="Json schema file path"), + data_file: str = Option( + None, + "-d", + "--data", + help="Json data file path. if not passed will try to read from stdin", + ), + quiet: bool = Option(False, "-q", "--quiet", help="Quiet mode, only print errors"), + key_hints: str = Option( + None, + "-k", + "--key-hints", + help="Extra prompt for the ai to help it complete the task", + ), +): + """ + Validate and translate a json schema to jq filter + """ + with open(schema_file) as f: + output_schema = json.load(f) + if data_file is None: + if sys.stdin.isatty(): + sys.exit("Error: No data piped to stdin.") + else: + if not quiet: + print("--data not provided, reading from stdin") + data_file = sys.stdin.read() + input_json = json.loads(data_file) + else: + with open(data_file) as f: + input_json = json.load(f) + + query = translate_schema( + output_schema=output_schema, + input_json=input_json, + key_hints=key_hints, + quiet=quiet, + ) + print(query) + + +def main(): + typer_app() + + +if __name__ == "__main__": + main() diff --git a/jaiqu/jaiqu.py b/jaiqu/jaiqu.py index abe8841..57c2210 100644 --- a/jaiqu/jaiqu.py +++ b/jaiqu/jaiqu.py @@ -6,7 +6,7 @@ from .helpers import identify_key, create_jq_string, repair_query, dict_to_jq_filter -def validate_schema(input_json: dict, output_schema: dict, openai_api_key: str | None = None, key_hints=None) -> tuple[dict, bool]: +def validate_schema(input_json: dict, output_schema: dict, openai_api_key: str | None = None, key_hints=None, quiet=False) -> tuple[dict, bool]: """Validates the schema of the input JSON against the output schema. Args: input_json (dict): The input JSON parsed into a dictionary. @@ -20,7 +20,7 @@ def validate_schema(input_json: dict, output_schema: dict, openai_api_key: str | results = {} valid = True - with tqdm(total=len(output_schema['properties']), desc="Validating schema") as pbar: + with tqdm(total=len(output_schema['properties']), desc="Validating schema", disable=quiet) as pbar: for key, value in output_schema['properties'].items(): pbar.set_postfix_str(f"Key: {key}", refresh=True) response_key, response_reasoning = identify_key(key, value, input_json, openai_api_key, key_hints) @@ -44,7 +44,7 @@ def validate_schema(input_json: dict, output_schema: dict, openai_api_key: str | return results, valid -def translate_schema(input_json, output_schema, openai_api_key: str | None = None, key_hints=None, max_retries=10) -> str: +def translate_schema(input_json, output_schema, openai_api_key: str | None = None, key_hints=None, max_retries=10, quiet=False) -> str: """ Translate the input JSON schema into a filtering query using jq. @@ -64,7 +64,7 @@ def translate_schema(input_json, output_schema, openai_api_key: str | None = Non RuntimeError: If failed to validate the jq filter after maximum retries. """ - schema_properties, is_valid = validate_schema(input_json, output_schema, key_hints=key_hints, openai_api_key=openai_api_key) + schema_properties, is_valid = validate_schema(input_json, output_schema, key_hints=key_hints, openai_api_key=openai_api_key, quiet=quiet) if not is_valid: raise RuntimeError( f"The input JSON does not contain the required data to satisfy the output schema: \n\n{json.dumps(schema_properties, indent=2)}") @@ -73,7 +73,7 @@ def translate_schema(input_json, output_schema, openai_api_key: str | None = Non filter_query = {} - with tqdm(total=len(filtered_schema), desc="Translating schema") as pbar, tqdm(total=max_retries, desc="Retry attempts") as pbar_retries: + with tqdm(total=len(filtered_schema), desc="Translating schema", disable=quiet) as pbar, tqdm(total=max_retries, desc="Retry attempts", disable=quiet) as pbar_retries: for key, value in filtered_schema.items(): pbar.set_postfix_str(f"Key: {key}", refresh=True) jq_string = create_jq_string(input_json, key, value, openai_api_key) @@ -101,7 +101,7 @@ def translate_schema(input_json, output_schema, openai_api_key: str | None = Non complete_filter = dict_to_jq_filter(filter_query) # Validate JSON tries = 0 - with tqdm(total=max_retries, desc="Validation attempts") as pbar_validation: + with tqdm(total=max_retries, desc="Validation attempts", disable=quiet) as pbar_validation: while True: try: result = jq.compile(complete_filter).input(input_json).all()[0] diff --git a/pyproject.toml b/pyproject.toml index 56fc039..7d04d65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,8 @@ classifiers = [ dependencies = [ "jq==1.6.0", "openai~=1.12.0", - "jsonschema==4.21.1" + "jsonschema==4.21.1", + "typer==0.9.0", ] [project.optional-dependencies] @@ -34,3 +35,6 @@ dev = [ [project.urls] Homepage = "https://github.com/AgentOps-AI/Jaiqu" Issues = "https://github.com/AgentOps-AI/Jaiqu/issues" + +[project.entry-points.console_scripts] +jaiqu = "jaiqu.cli:main" \ No newline at end of file diff --git a/samples/data.json b/samples/data.json new file mode 100644 index 0000000..8e15fe8 --- /dev/null +++ b/samples/data.json @@ -0,0 +1,11 @@ +{ + "call.id": "123", + "datetime": "2022-01-01", + "timestamp": 1640995200, + "Address": "123 Main St", + "user": { + "name": "John Doe", + "age": 30, + "contact": "john@email.com" + } +} diff --git a/samples/schema.json b/samples/schema.json new file mode 100644 index 0000000..7b1b7bc --- /dev/null +++ b/samples/schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["string", "null"], + "description": "A unique identifier for the record." + }, + "date": { + "type": "string", + "description": "A string describing the date." + }, + "model": { + "type": "string", + "description": "A text field representing the model used." + } + }, + "required": [ + "id", + "date" + ] +} From 092a5fded86e94d19c5e34de3b17a7927419bba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Adriano?= Date: Mon, 1 Apr 2024 19:45:44 +0200 Subject: [PATCH 2/5] Add typer to requirements.txt as well --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dbdce18..5f10958 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ jq==1.6.0 openai>=1.12.0,<2.0.0 -jsonschema==4.21.1 \ No newline at end of file +jsonschema==4.21.1 +typer==0.9.0 \ No newline at end of file From 77ef2d54d7d24a6ad6388cf7b0aa09799057871e Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 1 Apr 2024 12:44:10 -0700 Subject: [PATCH 3/5] fix setuptools --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7d04d65..ac596bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,4 +37,7 @@ Homepage = "https://github.com/AgentOps-AI/Jaiqu" Issues = "https://github.com/AgentOps-AI/Jaiqu/issues" [project.entry-points.console_scripts] -jaiqu = "jaiqu.cli:main" \ No newline at end of file +jaiqu = "jaiqu.cli:main" + +[tool.setuptools] +packages = { find = { where = ["."], exclude = ["samples"] } } \ No newline at end of file From 5e872702bde8c4304a6890f923940401471f9233 Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 1 Apr 2024 12:55:31 -0700 Subject: [PATCH 4/5] update args --- jaiqu/cli.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/jaiqu/cli.py b/jaiqu/cli.py index 11cf2ec..44bf3b5 100644 --- a/jaiqu/cli.py +++ b/jaiqu/cli.py @@ -24,6 +24,12 @@ def jaiqu( "--key-hints", help="Extra prompt for the ai to help it complete the task", ), + max_retries: int = Option( + 10, + "-r", + "--max-retries", + help="Max number of retries for the ai to complete the task", + ), ): """ Validate and translate a json schema to jq filter @@ -46,7 +52,8 @@ def jaiqu( output_schema=output_schema, input_json=input_json, key_hints=key_hints, - quiet=quiet, + max_retries=max_retries, + quiet=quiet ) print(query) From 211ac84b23ba5e7f7f76176d8f672168aaba6b27 Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 1 Apr 2024 12:56:38 -0700 Subject: [PATCH 5/5] verison bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ac596bc..dba106d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "jaiqu" -version = "0.0.4" +version = "0.0.5" authors = [ { name = "Alex Reibman", email = "areibman@gmail.com" }, { name = "Howard Gil", email = "howardbgil@gmail.com" },