reduce max_completion_tokens to 15 to speed up generation

nod-ai · Nov 15, 2024 · cbe008a · cbe008a
1 parent 64b758b
commit cbe008a
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/app_tests/integration_tests/llm/cpu_llm_server_test.py b/app_tests/integration_tests/llm/cpu_llm_server_test.py
@@ -37,7 +37,7 @@ def do_generate(prompt, port):
     # Create a GenerateReqInput-like structure
     data = {
         "text": prompt,
-        "sampling_params": {"max_completion_tokens": 50, "temperature": 0.7},
+        "sampling_params": {"max_completion_tokens": 15, "temperature": 0.7},
         "rid": uuid.uuid4().hex,
         "return_logprob": False,
         "logprob_start_len": -1,