Allow async

h2oai · Nov 1, 2024 · 86e631d · 86e631d
1 parent 89d4051
commit 86e631d
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/src/gpt_langchain.py b/src/gpt_langchain.py
@@ -3349,7 +3349,7 @@ def get_llm(use_openai_model=False,
                     else:
                         # for vllm 0.6.3+
                         kwargs_extra.update(dict(response_format=dict(type='text')))
-                async_output = False  # https://github.com/h2oai/h2ogpt/issues/928
+                # async_output = False  # https://github.com/h2oai/h2ogpt/issues/928
                 # async_sem = asyncio.Semaphore(num_async) if async_output else NullContext()
                 kwargs_extra.update(dict(openai_api_key=api_key,
                                          # batch_size=1,
@@ -3409,7 +3409,7 @@ def get_llm(use_openai_model=False,
                                          user_prompt_for_fake_system_prompt=user_prompt_for_fake_system_prompt,
                                          openai_api_base=base_url,
                                          openai_api_key=api_key,
-                                         batch_size=1,  # https://github.com/h2oai/h2ogpt/issues/928
+                                         batch_size=num_async if async_output else 1,
                                          client=openai_client_completions,
                                          async_client=openai_async_client_completions,
                                          async_sem=async_sem,

diff --git a/src/version.py b/src/version.py
@@ -1 +1 @@
-__version__ = "1f0a8f36fd8a811958eef2c671bc343d77a187fb"
+__version__ = "89d4051e6818a70033874c920eb5087d7c7c81dc"
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		__version__ = "1f0a8f36fd8a811958eef2c671bc343d77a187fb"
		__version__ = "89d4051e6818a70033874c920eb5087d7c7c81dc"