From 55a56ac373c4a3745e073c1575d5ebe342259fad Mon Sep 17 00:00:00 2001
From: Huanzhi Mao <huanzhimao@gmail.com>
Date: Thu, 1 Aug 2024 15:25:58 -0700
Subject: [PATCH 01/18] create bfcl sub-folder

---
 .../eval_checker/api_status_check_ground_truth_REST.json          | 0
 .../eval_checker/api_status_check_ground_truth_executable.json    | 0
 .../{ => bfcl}/eval_checker/checker.py                            | 0
 .../{ => bfcl}/eval_checker/custom_exception.py                   | 0
 .../{ => bfcl}/eval_checker/eval_checker_constant.py              | 0
 .../{ => bfcl}/eval_checker/eval_runner.py                        | 0
 .../{ => bfcl}/eval_checker/eval_runner_helper.py                 | 0
 .../{ => bfcl}/eval_checker/executable_python_function.py         | 0
 .../{ => bfcl}/eval_checker/java_type_converter.py                | 0
 .../{ => bfcl}/eval_checker/js_type_converter.py                  | 0
 .../{ => bfcl}/eval_checker/rest-eval-response_v5.jsonl           | 0
 .../{ => bfcl}/model_handler/arctic_handler.py                    | 0
 .../{ => bfcl}/model_handler/claude_fc_handler.py                 | 0
 .../{ => bfcl}/model_handler/claude_prompt_handler.py             | 0
 .../{ => bfcl}/model_handler/cohere_handler.py                    | 0
 .../{ => bfcl}/model_handler/constant.py                          | 0
 .../{ => bfcl}/model_handler/databricks_handler.py                | 0
 .../{ => bfcl}/model_handler/deepseek_handler.py                  | 0
 .../{ => bfcl}/model_handler/firework_ai_handler.py               | 0
 .../{ => bfcl}/model_handler/functionary_handler.py               | 0
 .../{ => bfcl}/model_handler/gemini_handler.py                    | 0
 .../{ => bfcl}/model_handler/gemma_handler.py                     | 0
 .../{ => bfcl}/model_handler/glaive_handler.py                    | 0
 .../{ => bfcl}/model_handler/glm_handler.py                       | 0
 .../{ => bfcl}/model_handler/gorilla_handler.py                   | 0
 .../{ => bfcl}/model_handler/gpt_handler.py                       | 0
 .../{ => bfcl}/model_handler/granite_handler.py                   | 0
 .../{ => bfcl}/model_handler/handler.py                           | 0
 .../{ => bfcl}/model_handler/handler_map.py                       | 0
 .../{ => bfcl}/model_handler/hermes_handler.py                    | 0
 .../{ => bfcl}/model_handler/java_parser.py                       | 0
 .../{ => bfcl}/model_handler/js_parser.py                         | 0
 .../{ => bfcl}/model_handler/llama_handler.py                     | 0
 .../{ => bfcl}/model_handler/mistral_handler.py                   | 0
 .../{ => bfcl}/model_handler/model_style.py                       | 0
 .../{ => bfcl}/model_handler/nexus_handler.py                     | 0
 .../{ => bfcl}/model_handler/nvidia_handler.py                    | 0
 .../{ => bfcl}/model_handler/oss_handler.py                       | 0
 .../{ => bfcl}/model_handler/utils.py                             | 0
 .../{ => bfcl}/model_handler/xlam_handler.py                      | 0
 .../{ => bfcl}/model_handler/yi_handler.py                        | 0
 41 files changed, 0 insertions(+), 0 deletions(-)
 rename berkeley-function-call-leaderboard/{ => bfcl}/eval_checker/api_status_check_ground_truth_REST.json (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/eval_checker/api_status_check_ground_truth_executable.json (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/eval_checker/checker.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/eval_checker/custom_exception.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/eval_checker/eval_checker_constant.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/eval_checker/eval_runner.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/eval_checker/eval_runner_helper.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/eval_checker/executable_python_function.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/eval_checker/java_type_converter.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/eval_checker/js_type_converter.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/eval_checker/rest-eval-response_v5.jsonl (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/arctic_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/claude_fc_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/claude_prompt_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/cohere_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/constant.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/databricks_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/deepseek_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/firework_ai_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/functionary_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/gemini_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/gemma_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/glaive_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/glm_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/gorilla_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/gpt_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/granite_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/handler_map.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/hermes_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/java_parser.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/js_parser.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/llama_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/mistral_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/model_style.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/nexus_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/nvidia_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/oss_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/utils.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/xlam_handler.py (100%)
 rename berkeley-function-call-leaderboard/{ => bfcl}/model_handler/yi_handler.py (100%)

diff --git a/berkeley-function-call-leaderboard/eval_checker/api_status_check_ground_truth_REST.json b/berkeley-function-call-leaderboard/bfcl/eval_checker/api_status_check_ground_truth_REST.json
similarity index 100%
rename from berkeley-function-call-leaderboard/eval_checker/api_status_check_ground_truth_REST.json
rename to berkeley-function-call-leaderboard/bfcl/eval_checker/api_status_check_ground_truth_REST.json
diff --git a/berkeley-function-call-leaderboard/eval_checker/api_status_check_ground_truth_executable.json b/berkeley-function-call-leaderboard/bfcl/eval_checker/api_status_check_ground_truth_executable.json
similarity index 100%
rename from berkeley-function-call-leaderboard/eval_checker/api_status_check_ground_truth_executable.json
rename to berkeley-function-call-leaderboard/bfcl/eval_checker/api_status_check_ground_truth_executable.json
diff --git a/berkeley-function-call-leaderboard/eval_checker/checker.py b/berkeley-function-call-leaderboard/bfcl/eval_checker/checker.py
similarity index 100%
rename from berkeley-function-call-leaderboard/eval_checker/checker.py
rename to berkeley-function-call-leaderboard/bfcl/eval_checker/checker.py
diff --git a/berkeley-function-call-leaderboard/eval_checker/custom_exception.py b/berkeley-function-call-leaderboard/bfcl/eval_checker/custom_exception.py
similarity index 100%
rename from berkeley-function-call-leaderboard/eval_checker/custom_exception.py
rename to berkeley-function-call-leaderboard/bfcl/eval_checker/custom_exception.py
diff --git a/berkeley-function-call-leaderboard/eval_checker/eval_checker_constant.py b/berkeley-function-call-leaderboard/bfcl/eval_checker/eval_checker_constant.py
similarity index 100%
rename from berkeley-function-call-leaderboard/eval_checker/eval_checker_constant.py
rename to berkeley-function-call-leaderboard/bfcl/eval_checker/eval_checker_constant.py
diff --git a/berkeley-function-call-leaderboard/eval_checker/eval_runner.py b/berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner.py
similarity index 100%
rename from berkeley-function-call-leaderboard/eval_checker/eval_runner.py
rename to berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner.py
diff --git a/berkeley-function-call-leaderboard/eval_checker/eval_runner_helper.py b/berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner_helper.py
similarity index 100%
rename from berkeley-function-call-leaderboard/eval_checker/eval_runner_helper.py
rename to berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner_helper.py
diff --git a/berkeley-function-call-leaderboard/eval_checker/executable_python_function.py b/berkeley-function-call-leaderboard/bfcl/eval_checker/executable_python_function.py
similarity index 100%
rename from berkeley-function-call-leaderboard/eval_checker/executable_python_function.py
rename to berkeley-function-call-leaderboard/bfcl/eval_checker/executable_python_function.py
diff --git a/berkeley-function-call-leaderboard/eval_checker/java_type_converter.py b/berkeley-function-call-leaderboard/bfcl/eval_checker/java_type_converter.py
similarity index 100%
rename from berkeley-function-call-leaderboard/eval_checker/java_type_converter.py
rename to berkeley-function-call-leaderboard/bfcl/eval_checker/java_type_converter.py
diff --git a/berkeley-function-call-leaderboard/eval_checker/js_type_converter.py b/berkeley-function-call-leaderboard/bfcl/eval_checker/js_type_converter.py
similarity index 100%
rename from berkeley-function-call-leaderboard/eval_checker/js_type_converter.py
rename to berkeley-function-call-leaderboard/bfcl/eval_checker/js_type_converter.py
diff --git a/berkeley-function-call-leaderboard/eval_checker/rest-eval-response_v5.jsonl b/berkeley-function-call-leaderboard/bfcl/eval_checker/rest-eval-response_v5.jsonl
similarity index 100%
rename from berkeley-function-call-leaderboard/eval_checker/rest-eval-response_v5.jsonl
rename to berkeley-function-call-leaderboard/bfcl/eval_checker/rest-eval-response_v5.jsonl
diff --git a/berkeley-function-call-leaderboard/model_handler/arctic_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/arctic_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/arctic_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/arctic_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/claude_fc_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/claude_fc_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/claude_fc_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/claude_fc_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/claude_prompt_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/claude_prompt_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/claude_prompt_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/claude_prompt_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/cohere_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/cohere_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/cohere_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/cohere_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/constant.py b/berkeley-function-call-leaderboard/bfcl/model_handler/constant.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/constant.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/constant.py
diff --git a/berkeley-function-call-leaderboard/model_handler/databricks_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/databricks_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/databricks_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/databricks_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/deepseek_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/deepseek_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/deepseek_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/deepseek_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/firework_ai_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/firework_ai_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/firework_ai_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/firework_ai_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/functionary_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/functionary_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/functionary_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/functionary_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/gemini_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/gemini_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/gemini_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/gemini_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/gemma_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/gemma_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/gemma_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/gemma_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/glaive_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/glaive_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/glaive_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/glaive_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/glm_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/glm_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/glm_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/glm_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/gorilla_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/gorilla_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/gorilla_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/gorilla_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/gpt_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/gpt_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/gpt_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/gpt_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/granite_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/granite_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/granite_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/granite_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/handler_map.py b/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/handler_map.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py
diff --git a/berkeley-function-call-leaderboard/model_handler/hermes_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/hermes_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/hermes_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/hermes_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/java_parser.py b/berkeley-function-call-leaderboard/bfcl/model_handler/java_parser.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/java_parser.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/java_parser.py
diff --git a/berkeley-function-call-leaderboard/model_handler/js_parser.py b/berkeley-function-call-leaderboard/bfcl/model_handler/js_parser.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/js_parser.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/js_parser.py
diff --git a/berkeley-function-call-leaderboard/model_handler/llama_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/llama_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/llama_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/llama_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/mistral_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/mistral_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/mistral_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/mistral_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/model_style.py b/berkeley-function-call-leaderboard/bfcl/model_handler/model_style.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/model_style.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/model_style.py
diff --git a/berkeley-function-call-leaderboard/model_handler/nexus_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/nexus_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/nexus_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/nexus_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/nvidia_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/nvidia_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/nvidia_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/nvidia_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/oss_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/oss_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/oss_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/oss_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/utils.py b/berkeley-function-call-leaderboard/bfcl/model_handler/utils.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/utils.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/utils.py
diff --git a/berkeley-function-call-leaderboard/model_handler/xlam_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/xlam_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/xlam_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/xlam_handler.py
diff --git a/berkeley-function-call-leaderboard/model_handler/yi_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/yi_handler.py
similarity index 100%
rename from berkeley-function-call-leaderboard/model_handler/yi_handler.py
rename to berkeley-function-call-leaderboard/bfcl/model_handler/yi_handler.py

From 598604d06c1b55b1d05588eacb88b4168c67c7d5 Mon Sep 17 00:00:00 2001
From: Huanzhi Mao <huanzhimao@gmail.com>
Date: Thu, 1 Aug 2024 15:29:08 -0700
Subject: [PATCH 02/18] add setup.py

---
 berkeley-function-call-leaderboard/setup.py | 39 +++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 berkeley-function-call-leaderboard/setup.py

diff --git a/berkeley-function-call-leaderboard/setup.py b/berkeley-function-call-leaderboard/setup.py
new file mode 100644
index 000000000..863ef21b5
--- /dev/null
+++ b/berkeley-function-call-leaderboard/setup.py
@@ -0,0 +1,39 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="bfcl",
+    version="0.1.0",
+    description="Berkeley Function Calling Leaderboard (BFCL)",
+    author="Sky Computing Lab",
+    author_email="sky@cs.berkeley.edu",
+    long_description=open("README.md").read(),
+    long_description_content_type="text/markdown",
+    python_requires=">=3.9",
+    license="Apache 2.0",
+    packages=find_packages(include=["bfcl*"]),
+    install_requires=[
+        "requests",
+        "tqdm",
+        "numpy==1.26.4",
+        "pandas",
+        "huggingface_hub",
+        "pydantic>=2.8.2",
+        "python-dotenv>=1.0.1",
+        "tree_sitter==0.21.3",
+        "tree-sitter-java==0.21.0",
+        "tree-sitter-javascript==0.21.4",
+        "openai==1.35.13",
+    ],
+    extras_require={
+        "oss_eval": ["vllm==0.5.0"],
+        "proprietary_eval": [
+            "mistralai==0.4.2",
+            "anthropic==0.31.1",
+            "cohere==5.5.8",
+        ],
+        "all": ["bfcl[oss_eval]", "bfcl[proprietary_eval]"],
+    },
+    project_urls={
+        "Repository": "https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard",
+    },
+)

From 16b728aeb207e78927223680427039b0b563ccd7 Mon Sep 17 00:00:00 2001
From: Huanzhi Mao <huanzhimao@gmail.com>
Date: Thu, 1 Aug 2024 15:29:59 -0700
Subject: [PATCH 03/18] add __init__.py

---
 berkeley-function-call-leaderboard/bfcl/__init__.py               | 0
 berkeley-function-call-leaderboard/bfcl/eval_checker/__init__.py  | 0
 berkeley-function-call-leaderboard/bfcl/model_handler/__init__.py | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 berkeley-function-call-leaderboard/bfcl/__init__.py
 create mode 100644 berkeley-function-call-leaderboard/bfcl/eval_checker/__init__.py
 create mode 100644 berkeley-function-call-leaderboard/bfcl/model_handler/__init__.py

diff --git a/berkeley-function-call-leaderboard/bfcl/__init__.py b/berkeley-function-call-leaderboard/bfcl/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/berkeley-function-call-leaderboard/bfcl/eval_checker/__init__.py b/berkeley-function-call-leaderboard/bfcl/eval_checker/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/__init__.py b/berkeley-function-call-leaderboard/bfcl/model_handler/__init__.py
new file mode 100644
index 000000000..e69de29bb

From a33afd9d7a90cbc90c22615a164e90d485cbabfe Mon Sep 17 00:00:00 2001
From: Huanzhi Mao <huanzhimao@gmail.com>
Date: Thu, 1 Aug 2024 15:38:40 -0700
Subject: [PATCH 04/18] Add pyproject.toml file

---
 berkeley-function-call-leaderboard/pyproject.toml | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 berkeley-function-call-leaderboard/pyproject.toml

diff --git a/berkeley-function-call-leaderboard/pyproject.toml b/berkeley-function-call-leaderboard/pyproject.toml
new file mode 100644
index 000000000..2f2101195
--- /dev/null
+++ b/berkeley-function-call-leaderboard/pyproject.toml
@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=40.8.0", "wheel"]
+build-backend = "setuptools.build_meta"

From 7dfca12f41c151588def60437e9bc4cc3056245f Mon Sep 17 00:00:00 2001
From: Huanzhi Mao <huanzhimao@gmail.com>
Date: Thu, 1 Aug 2024 15:47:14 -0700
Subject: [PATCH 05/18] update path

---
 .../apply_function_credential_config.py              |  2 +-
 .../bfcl/eval_checker/eval_runner.py                 | 12 ++++--------
 .../bfcl/eval_checker/eval_runner_helper.py          |  4 ++--
 .../bfcl/eval_checker/executable_python_function.py  |  2 +-
 .../bfcl/eval_checker/java_type_converter.py         |  2 +-
 .../bfcl/eval_checker/js_type_converter.py           |  2 +-
 .../openfunctions_evaluation.py                      |  8 ++++----
 7 files changed, 14 insertions(+), 18 deletions(-)

diff --git a/berkeley-function-call-leaderboard/apply_function_credential_config.py b/berkeley-function-call-leaderboard/apply_function_credential_config.py
index 8646d5c29..23fa53fdc 100644
--- a/berkeley-function-call-leaderboard/apply_function_credential_config.py
+++ b/berkeley-function-call-leaderboard/apply_function_credential_config.py
@@ -2,7 +2,7 @@
 import json
 import argparse
 import os
-from eval_checker import custom_exception
+from bfcl.eval_checker import custom_exception
 
 parser = argparse.ArgumentParser(description="Replace placeholders in the function credential config file.")
 parser.add_argument("--input-path", help="Path to the function credential config file. Can be a file or a directory.")
diff --git a/berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner.py b/berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner.py
index 1e0f0645e..2a2513439 100644
--- a/berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner.py
+++ b/berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner.py
@@ -1,7 +1,3 @@
-import sys
-
-sys.path.append("../")
-
 from checker import ast_checker, exec_checker, executable_checker_rest
 from custom_exception import BadAPIStatusError
 from eval_runner_helper import *
@@ -417,10 +413,10 @@ def runner(model_names, test_categories, api_sanity_check):
     print(f"🏁 Evaluation completed. See {os.path.abspath(OUTPUT_PATH + 'data.csv')} for evaluation results.")
 
 
-INPUT_PATH = "../result/"
-PROMPT_PATH = "../data/"
-POSSIBLE_ANSWER_PATH = "../data/possible_answer/"
-OUTPUT_PATH = "../score/"
+INPUT_PATH = "../../result/"
+PROMPT_PATH = "../../data/"
+POSSIBLE_ANSWER_PATH = "../../data/possible_answer/"
+OUTPUT_PATH = "../../score/"
 
 # A dictionary to store the results
 # Key is model name, value is a dictionary with keys as test category and values as a dictionary with accuracy and total count
diff --git a/berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner_helper.py b/berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner_helper.py
index dc72f74cb..54d2b6f7f 100644
--- a/berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner_helper.py
+++ b/berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner_helper.py
@@ -6,7 +6,7 @@
 
 import numpy as np
 from custom_exception import BadAPIStatusError
-from model_handler.handler_map import handler_map
+from bfcl.model_handler.handler_map import handler_map
 from tqdm import tqdm
 
 REST_API_GROUND_TRUTH_FILE_PATH = "api_status_check_ground_truth_REST.json"
@@ -670,7 +670,7 @@ def api_status_sanity_check_rest():
     ground_truth_dummy = load_file(REST_API_GROUND_TRUTH_FILE_PATH)
 
     # Use the ground truth data to make sure the API is working correctly
-    command = f"cd .. ; python apply_function_credential_config.py --input-path ./eval_checker/{REST_API_GROUND_TRUTH_FILE_PATH};"
+    command = f"cd ../.. ; python apply_function_credential_config.py --input-path ./bfcl/eval_checker/{REST_API_GROUND_TRUTH_FILE_PATH};"
     try:
         subprocess.run(command, shell=True, capture_output=True, text=True, check=True)
     except subprocess.CalledProcessError as e:
diff --git a/berkeley-function-call-leaderboard/bfcl/eval_checker/executable_python_function.py b/berkeley-function-call-leaderboard/bfcl/eval_checker/executable_python_function.py
index bd3af9ad2..324110851 100644
--- a/berkeley-function-call-leaderboard/bfcl/eval_checker/executable_python_function.py
+++ b/berkeley-function-call-leaderboard/bfcl/eval_checker/executable_python_function.py
@@ -5,7 +5,7 @@
 import time
 
 api_key = {}
-with open("../function_credential_config.json") as f:
+with open("../../function_credential_config.json") as f:
     data = json.loads(f.read())
     for item in data:
         for k, v in item.items():
diff --git a/berkeley-function-call-leaderboard/bfcl/eval_checker/java_type_converter.py b/berkeley-function-call-leaderboard/bfcl/eval_checker/java_type_converter.py
index 973aaa0e2..7a2a5210a 100644
--- a/berkeley-function-call-leaderboard/bfcl/eval_checker/java_type_converter.py
+++ b/berkeley-function-call-leaderboard/bfcl/eval_checker/java_type_converter.py
@@ -1,6 +1,6 @@
 import re
 from typing import List, Dict, Union
-from model_handler.constant import JAVA_TYPE_CONVERSION
+from bfcl.model_handler.constant import JAVA_TYPE_CONVERSION
 
 
 def java_type_converter(value, expected_type, nested_type=None):
diff --git a/berkeley-function-call-leaderboard/bfcl/eval_checker/js_type_converter.py b/berkeley-function-call-leaderboard/bfcl/eval_checker/js_type_converter.py
index 2619494ca..51571a226 100644
--- a/berkeley-function-call-leaderboard/bfcl/eval_checker/js_type_converter.py
+++ b/berkeley-function-call-leaderboard/bfcl/eval_checker/js_type_converter.py
@@ -1,5 +1,5 @@
 import re
-from model_handler.constant import JS_TYPE_CONVERSION
+from bfcl.model_handler.constant import JS_TYPE_CONVERSION
 
 
 def js_type_converter(value, expected_type, nested_type=None):
diff --git a/berkeley-function-call-leaderboard/openfunctions_evaluation.py b/berkeley-function-call-leaderboard/openfunctions_evaluation.py
index 1c4ba9c38..82099f379 100644
--- a/berkeley-function-call-leaderboard/openfunctions_evaluation.py
+++ b/berkeley-function-call-leaderboard/openfunctions_evaluation.py
@@ -1,9 +1,9 @@
 import argparse, json, os
 from tqdm import tqdm
-from model_handler.handler_map import handler_map
-from model_handler.model_style import ModelStyle
-from model_handler.constant import USE_COHERE_OPTIMIZATION
-from eval_checker.eval_checker_constant import TEST_COLLECTION_MAPPING
+from bfcl.model_handler.handler_map import handler_map
+from bfcl.model_handler.model_style import ModelStyle
+from bfcl.model_handler.constant import USE_COHERE_OPTIMIZATION
+from bfcl.eval_checker.eval_checker_constant import TEST_COLLECTION_MAPPING
 
 
 def get_args():

From 092f20b5631bd4f21a7bd82c48cf970364ad79fa Mon Sep 17 00:00:00 2001
From: Huanzhi Mao <huanzhimao@gmail.com>
Date: Thu, 1 Aug 2024 15:47:31 -0700
Subject: [PATCH 06/18] remove unnecessary write function

---
 .../bfcl/model_handler/firework_ai_handler.py       | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/firework_ai_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/firework_ai_handler.py
index ac2aa7062..b018251f2 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/firework_ai_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/firework_ai_handler.py
@@ -20,19 +20,6 @@ def __init__(self, model_name, temperature=0.0, top_p=1, max_tokens=1000) -> Non
             api_key=os.getenv("FIRE_WORKS_API_KEY"),
         )
 
-    def write(self, result, file_to_open):
-        # This method is used to write the result to the file.
-        if not os.path.exists("./result"):
-            os.mkdir("./result")
-        if not os.path.exists(f"./result/{self.model_name}"):
-            os.mkdir(f"./result/{self.model_name}")
-        with open(
-            f"./result/{self.model_name}/"
-            + file_to_open.replace(".json", "_result.json"),
-            "a+",
-        ) as f:
-            f.write(json.dumps(result) + "\n")
-
     def inference(self, prompt, functions, test_category):
         functions = language_specific_pre_processing(functions, test_category)
         if type(functions) is not list:

From ab2769a24a78d6a213e3324e368af96e35f0207c Mon Sep 17 00:00:00 2001
From: Huanzhi Mao <huanzhimao@gmail.com>
Date: Thu, 1 Aug 2024 15:49:44 -0700
Subject: [PATCH 07/18] update path for model handlers

---
 .../bfcl/eval_checker/checker.py              |  2 +-
 .../bfcl/model_handler/arctic_handler.py      |  4 +-
 .../bfcl/model_handler/claude_fc_handler.py   | 10 ++--
 .../model_handler/claude_prompt_handler.py    |  8 ++--
 .../bfcl/model_handler/cohere_handler.py      | 10 ++--
 .../bfcl/model_handler/databricks_handler.py  |  8 ++--
 .../bfcl/model_handler/deepseek_handler.py    |  4 +-
 .../bfcl/model_handler/firework_ai_handler.py |  8 ++--
 .../bfcl/model_handler/functionary_handler.py |  4 +-
 .../bfcl/model_handler/gemini_handler.py      |  8 ++--
 .../bfcl/model_handler/gemma_handler.py       |  4 +-
 .../bfcl/model_handler/glaive_handler.py      |  4 +-
 .../bfcl/model_handler/glm_handler.py         | 12 ++---
 .../bfcl/model_handler/gorilla_handler.py     |  6 +--
 .../bfcl/model_handler/gpt_handler.py         |  8 ++--
 .../bfcl/model_handler/granite_handler.py     |  8 ++--
 .../bfcl/model_handler/handler.py             |  2 +-
 .../bfcl/model_handler/handler_map.py         | 46 +++++++++----------
 .../bfcl/model_handler/hermes_handler.py      |  8 ++--
 .../bfcl/model_handler/llama_handler.py       |  6 +--
 .../bfcl/model_handler/mistral_handler.py     |  8 ++--
 .../bfcl/model_handler/nexus_handler.py       |  6 +--
 .../bfcl/model_handler/nvidia_handler.py      | 10 ++--
 .../bfcl/model_handler/oss_handler.py         |  6 +--
 .../bfcl/model_handler/utils.py               |  8 ++--
 .../bfcl/model_handler/xlam_handler.py        |  4 +-
 .../bfcl/model_handler/yi_handler.py          |  8 ++--
 27 files changed, 110 insertions(+), 110 deletions(-)

diff --git a/berkeley-function-call-leaderboard/bfcl/eval_checker/checker.py b/berkeley-function-call-leaderboard/bfcl/eval_checker/checker.py
index 3d8025af0..c15b4abf7 100644
--- a/berkeley-function-call-leaderboard/bfcl/eval_checker/checker.py
+++ b/berkeley-function-call-leaderboard/bfcl/eval_checker/checker.py
@@ -1,4 +1,4 @@
-from model_handler.constant import (
+from bfcl.model_handler.constant import (
     UNDERSCORE_TO_DOT,
     JAVA_TYPE_CONVERSION,
     JS_TYPE_CONVERSION,
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/arctic_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/arctic_handler.py
index fdfd9d219..c1662e5ea 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/arctic_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/arctic_handler.py
@@ -1,5 +1,5 @@
-from model_handler.nvidia_handler import NvidiaHandler
-from model_handler.utils import ast_parse
+from bfcl.model_handler.nvidia_handler import NvidiaHandler
+from bfcl.model_handler.utils import ast_parse
 
 class ArcticHandler(NvidiaHandler):
     def __init__(self, model_name, temperature=0.7, top_p=1, max_tokens=1000) -> None:
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/claude_fc_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/claude_fc_handler.py
index 3b575d80e..ec2218991 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/claude_fc_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/claude_fc_handler.py
@@ -4,11 +4,11 @@
 
 from anthropic import Anthropic
 from anthropic.types import TextBlock, ToolUseBlock
-from model_handler.claude_prompt_handler import ClaudePromptingHandler
-from model_handler.constant import GORILLA_TO_OPENAPI
-from model_handler.handler import BaseHandler
-from model_handler.model_style import ModelStyle
-from model_handler.utils import (
+from bfcl.model_handler.claude_prompt_handler import ClaudePromptingHandler
+from bfcl.model_handler.constant import GORILLA_TO_OPENAPI
+from bfcl.model_handler.handler import BaseHandler
+from bfcl.model_handler.model_style import ModelStyle
+from bfcl.model_handler.utils import (
     ast_parse,
     augment_prompt_by_languge,
     convert_to_function_call,
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/claude_prompt_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/claude_prompt_handler.py
index ad5c180c2..ee6ad23a4 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/claude_prompt_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/claude_prompt_handler.py
@@ -1,6 +1,6 @@
-from model_handler.handler import BaseHandler
-from model_handler.model_style import ModelStyle
-from model_handler.utils import (
+from bfcl.model_handler.handler import BaseHandler
+from bfcl.model_handler.model_style import ModelStyle
+from bfcl.model_handler.utils import (
     convert_to_tool,
     ast_parse,
     augment_prompt_by_languge,
@@ -9,7 +9,7 @@
     _function_calls_valid_format_and_invoke_extraction,
     _convert_value,
 )
-from model_handler.constant import (
+from bfcl.model_handler.constant import (
     SYSTEM_PROMPT_FOR_CHAT_MODEL,
     USER_PROMPT_FOR_CHAT_MODEL,
     GORILLA_TO_PYTHON,
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/cohere_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/cohere_handler.py
index b573164bf..17da6bf3f 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/cohere_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/cohere_handler.py
@@ -1,15 +1,15 @@
 import os
 
-from model_handler.handler import BaseHandler
-from model_handler.model_style import ModelStyle
-from model_handler.utils import (
+from bfcl.model_handler.handler import BaseHandler
+from bfcl.model_handler.model_style import ModelStyle
+from bfcl.model_handler.utils import (
     augment_prompt_by_languge,
     language_specific_pre_processing,
     convert_to_tool,
     ast_parse,
     convert_to_function_call,
 )
-from model_handler.constant import (
+from bfcl.model_handler.constant import (
     SYSTEM_PROMPT_FOR_CHAT_MODEL,
     USER_PROMPT_FOR_CHAT_MODEL,
     GORILLA_TO_PYTHON,
@@ -17,7 +17,7 @@
 import time
 import cohere
 
-from model_handler.constant import USE_COHERE_OPTIMIZATION
+from bfcl.model_handler.constant import USE_COHERE_OPTIMIZATION
 
 
 class CohereHandler(BaseHandler):
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/databricks_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/databricks_handler.py
index 53474ca91..6d1948271 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/databricks_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/databricks_handler.py
@@ -1,7 +1,7 @@
-from model_handler.handler import BaseHandler
-from model_handler.model_style import ModelStyle
-from model_handler.utils import language_specific_pre_processing, ast_parse
-from model_handler.constant import (
+from bfcl.model_handler.handler import BaseHandler
+from bfcl.model_handler.model_style import ModelStyle
+from bfcl.model_handler.utils import language_specific_pre_processing, ast_parse
+from bfcl.model_handler.constant import (
     SYSTEM_PROMPT_FOR_CHAT_MODEL,
     USER_PROMPT_FOR_CHAT_MODEL,
     GORILLA_TO_OPENAPI,
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/deepseek_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/deepseek_handler.py
index 236c23273..ac942055b 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/deepseek_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/deepseek_handler.py
@@ -1,5 +1,5 @@
-from model_handler.oss_handler import OSSHandler
-from model_handler.utils import convert_to_function_call, ast_parse
+from bfcl.model_handler.oss_handler import OSSHandler
+from bfcl.model_handler.utils import convert_to_function_call, ast_parse
 import re
 
 
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/firework_ai_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/firework_ai_handler.py
index b018251f2..aaf315e3d 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/firework_ai_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/firework_ai_handler.py
@@ -2,10 +2,10 @@
 import os
 import time
 
-from model_handler.constant import GORILLA_TO_OPENAPI
-from model_handler.gpt_handler import OpenAIHandler
-from model_handler.model_style import ModelStyle
-from model_handler.utils import convert_to_tool, language_specific_pre_processing
+from bfcl.model_handler.constant import GORILLA_TO_OPENAPI
+from bfcl.model_handler.gpt_handler import OpenAIHandler
+from bfcl.model_handler.model_style import ModelStyle
+from bfcl.model_handler.utils import convert_to_tool, language_specific_pre_processing
 from openai import OpenAI
 
 
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/functionary_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/functionary_handler.py
index f7abbbe62..872ac2884 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/functionary_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/functionary_handler.py
@@ -1,5 +1,5 @@
-from model_handler.gpt_handler import OpenAIHandler
-from model_handler.model_style import ModelStyle
+from bfcl.model_handler.gpt_handler import OpenAIHandler
+from bfcl.model_handler.model_style import ModelStyle
 import os, json
 from openai import OpenAI
 
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/gemini_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/gemini_handler.py
index 80763fee1..7eb9fe9a6 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/gemini_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/gemini_handler.py
@@ -1,12 +1,12 @@
-from model_handler.handler import BaseHandler
-from model_handler.model_style import ModelStyle
-from model_handler.utils import (
+from bfcl.model_handler.handler import BaseHandler
+from bfcl.model_handler.model_style import ModelStyle
+from bfcl.model_handler.utils import (
     convert_to_tool,
     convert_to_function_call,
     augment_prompt_by_languge,
     language_specific_pre_processing,
 )
-from model_handler.constant import GORILLA_TO_OPENAPI
+from bfcl.model_handler.constant import GORILLA_TO_OPENAPI
 import subprocess, requests, json, time
 
 
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/gemma_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/gemma_handler.py
index d315801e4..d968ed274 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/gemma_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/gemma_handler.py
@@ -1,5 +1,5 @@
-from model_handler.oss_handler import OSSHandler
-from model_handler.utils import ast_parse
+from bfcl.model_handler.oss_handler import OSSHandler
+from bfcl.model_handler.utils import ast_parse
 import re
 
 
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/glaive_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/glaive_handler.py
index 1b68bf766..62cec4979 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/glaive_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/glaive_handler.py
@@ -1,5 +1,5 @@
-from model_handler.oss_handler import OSSHandler
-from model_handler.utils import convert_to_function_call
+from bfcl.model_handler.oss_handler import OSSHandler
+from bfcl.model_handler.utils import convert_to_function_call
 import json
 
 
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/glm_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/glm_handler.py
index c0475ce29..9f834758e 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/glm_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/glm_handler.py
@@ -1,17 +1,17 @@
-from model_handler.oss_handler import OSSHandler
-from model_handler.utils import ast_parse
-from model_handler.constant import (
+from bfcl.model_handler.oss_handler import OSSHandler
+from bfcl.model_handler.utils import ast_parse
+from bfcl.model_handler.constant import (
     SYSTEM_PROMPT_FOR_CHAT_MODEL,
     USER_PROMPT_FOR_CHAT_MODEL,
 )
-from model_handler.model_style import ModelStyle
-from model_handler.utils import (
+from bfcl.model_handler.model_style import ModelStyle
+from bfcl.model_handler.utils import (
     convert_to_tool,
     augment_prompt_by_languge,
     language_specific_pre_processing,
     convert_to_function_call,
 )
-from model_handler.constant import GORILLA_TO_OPENAPI
+from bfcl.model_handler.constant import GORILLA_TO_OPENAPI
 import json
 
 
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/gorilla_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/gorilla_handler.py
index 91a640c0a..87d96dceb 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/gorilla_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/gorilla_handler.py
@@ -1,6 +1,6 @@
-from model_handler.handler import BaseHandler
-from model_handler.model_style import ModelStyle
-from model_handler.utils import (
+from bfcl.model_handler.handler import BaseHandler
+from bfcl.model_handler.model_style import ModelStyle
+from bfcl.model_handler.utils import (
     ast_parse,
     augment_prompt_by_languge,
     language_specific_pre_processing,
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/gpt_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/gpt_handler.py
index 6449c948a..70384902c 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/gpt_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/gpt_handler.py
@@ -1,13 +1,13 @@
-from model_handler.handler import BaseHandler
-from model_handler.model_style import ModelStyle
-from model_handler.utils import (
+from bfcl.model_handler.handler import BaseHandler
+from bfcl.model_handler.model_style import ModelStyle
+from bfcl.model_handler.utils import (
     convert_to_tool,
     convert_to_function_call,
     augment_prompt_by_languge,
     language_specific_pre_processing,
     ast_parse,
 )
-from model_handler.constant import (
+from bfcl.model_handler.constant import (
     GORILLA_TO_OPENAPI,
     GORILLA_TO_PYTHON,
     USER_PROMPT_FOR_CHAT_MODEL,
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/granite_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/granite_handler.py
index b81780052..9e3bbbdba 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/granite_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/granite_handler.py
@@ -1,9 +1,9 @@
 import json
 
-from model_handler.model_style import ModelStyle
-from model_handler.oss_handler import OSSHandler
-from model_handler.constant import GORILLA_TO_OPENAPI
-from model_handler.utils import (
+from bfcl.model_handler.model_style import ModelStyle
+from bfcl.model_handler.oss_handler import OSSHandler
+from bfcl.model_handler.constant import GORILLA_TO_OPENAPI
+from bfcl.model_handler.utils import (
     language_specific_pre_processing,
     convert_to_tool,
     augment_prompt_by_languge,
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/handler.py
index a4af8cd2f..21dbef326 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/handler.py
@@ -1,4 +1,4 @@
-from model_handler.model_style import ModelStyle
+from bfcl.model_handler.model_style import ModelStyle
 import json, os
 
 
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py b/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py
index c825eaaa8..091c76e13 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py
@@ -1,26 +1,26 @@
-from model_handler.arctic_handler import ArcticHandler
-from model_handler.claude_fc_handler import ClaudeFCHandler
-from model_handler.claude_prompt_handler import ClaudePromptingHandler
-from model_handler.cohere_handler import CohereHandler
-from model_handler.databricks_handler import DatabricksHandler
-from model_handler.deepseek_handler import DeepseekHandler
-from model_handler.firework_ai_handler import FireworkAIHandler
-from model_handler.functionary_handler import FunctionaryHandler
-from model_handler.gemini_handler import GeminiHandler
-from model_handler.gemma_handler import GemmaHandler
-from model_handler.glaive_handler import GlaiveHandler
-from model_handler.gorilla_handler import GorillaHandler
-from model_handler.gpt_handler import OpenAIHandler
-from model_handler.hermes_handler import HermesHandler
-from model_handler.llama_handler import LlamaHandler
-from model_handler.mistral_handler import MistralHandler
-from model_handler.nexus_handler import NexusHandler
-from model_handler.oss_handler import OSSHandler
-from model_handler.granite_handler import GraniteHandler
-from model_handler.nvidia_handler import NvidiaHandler
-from model_handler.glm_handler import GLMHandler
-from model_handler.yi_handler import YiHandler
-from model_handler.xlam_handler import xLAMHandler
+from bfcl.model_handler.arctic_handler import ArcticHandler
+from bfcl.model_handler.claude_fc_handler import ClaudeFCHandler
+from bfcl.model_handler.claude_prompt_handler import ClaudePromptingHandler
+from bfcl.model_handler.cohere_handler import CohereHandler
+from bfcl.model_handler.databricks_handler import DatabricksHandler
+from bfcl.model_handler.deepseek_handler import DeepseekHandler
+from bfcl.model_handler.firework_ai_handler import FireworkAIHandler
+from bfcl.model_handler.functionary_handler import FunctionaryHandler
+from bfcl.model_handler.gemini_handler import GeminiHandler
+from bfcl.model_handler.gemma_handler import GemmaHandler
+from bfcl.model_handler.glaive_handler import GlaiveHandler
+from bfcl.model_handler.gorilla_handler import GorillaHandler
+from bfcl.model_handler.gpt_handler import OpenAIHandler
+from bfcl.model_handler.hermes_handler import HermesHandler
+from bfcl.model_handler.llama_handler import LlamaHandler
+from bfcl.model_handler.mistral_handler import MistralHandler
+from bfcl.model_handler.nexus_handler import NexusHandler
+from bfcl.model_handler.oss_handler import OSSHandler
+from bfcl.model_handler.granite_handler import GraniteHandler
+from bfcl.model_handler.nvidia_handler import NvidiaHandler
+from bfcl.model_handler.glm_handler import GLMHandler
+from bfcl.model_handler.yi_handler import YiHandler
+from bfcl.model_handler.xlam_handler import xLAMHandler
 
 handler_map = {
     "gorilla-openfunctions-v0": GorillaHandler,
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/hermes_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/hermes_handler.py
index df00afa38..c6d907555 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/hermes_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/hermes_handler.py
@@ -1,7 +1,7 @@
-from model_handler.oss_handler import OSSHandler
-from model_handler.utils import convert_to_tool
-from model_handler.constant import GORILLA_TO_OPENAPI
-from model_handler.model_style import ModelStyle
+from bfcl.model_handler.oss_handler import OSSHandler
+from bfcl.model_handler.utils import convert_to_tool
+from bfcl.model_handler.constant import GORILLA_TO_OPENAPI
+from bfcl.model_handler.model_style import ModelStyle
 import json
 
 
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/llama_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/llama_handler.py
index 06abac9b6..ae74b8486 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/llama_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/llama_handler.py
@@ -1,6 +1,6 @@
-from model_handler.oss_handler import OSSHandler
-from model_handler.utils import ast_parse
-from model_handler.constant import (
+from bfcl.model_handler.oss_handler import OSSHandler
+from bfcl.model_handler.utils import ast_parse
+from bfcl.model_handler.constant import (
     SYSTEM_PROMPT_FOR_CHAT_MODEL,
     USER_PROMPT_FOR_CHAT_MODEL,
 )
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/mistral_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/mistral_handler.py
index f6ce5bfb6..84256cc07 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/mistral_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/mistral_handler.py
@@ -1,11 +1,11 @@
-from model_handler.handler import BaseHandler
-from model_handler.model_style import ModelStyle
-from model_handler.constant import (
+from bfcl.model_handler.handler import BaseHandler
+from bfcl.model_handler.model_style import ModelStyle
+from bfcl.model_handler.constant import (
     SYSTEM_PROMPT_FOR_CHAT_MODEL,
     USER_PROMPT_FOR_CHAT_MODEL,
     GORILLA_TO_OPENAPI,
 )
-from model_handler.utils import (
+from bfcl.model_handler.utils import (
     convert_to_tool,
     ast_parse,
     convert_to_function_call,
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/nexus_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/nexus_handler.py
index 18b538c23..fece61412 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/nexus_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/nexus_handler.py
@@ -1,6 +1,6 @@
-from model_handler.model_style import ModelStyle
-from model_handler.handler import BaseHandler
-from model_handler.utils import (
+from bfcl.model_handler.model_style import ModelStyle
+from bfcl.model_handler.handler import BaseHandler
+from bfcl.model_handler.utils import (
     ast_parse,
     augment_prompt_by_languge,
     language_specific_pre_processing,
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/nvidia_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/nvidia_handler.py
index 80884de17..0e0cf33d5 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/nvidia_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/nvidia_handler.py
@@ -1,13 +1,13 @@
 import time,os,json
 from openai import OpenAI
-from model_handler.handler import BaseHandler
-from model_handler.model_style import ModelStyle
-from model_handler.utils import ast_parse
-from model_handler.utils import (
+from bfcl.model_handler.handler import BaseHandler
+from bfcl.model_handler.model_style import ModelStyle
+from bfcl.model_handler.utils import ast_parse
+from bfcl.model_handler.utils import (
     augment_prompt_by_languge,
     language_specific_pre_processing,
 )
-from model_handler.constant import (
+from bfcl.model_handler.constant import (
     USER_PROMPT_FOR_CHAT_MODEL,
     SYSTEM_PROMPT_FOR_CHAT_MODEL,
 )
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/oss_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/oss_handler.py
index 1859b3251..1b9c71e49 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/oss_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/oss_handler.py
@@ -1,9 +1,9 @@
 import json
 import os
 
-from model_handler.handler import BaseHandler
-from model_handler.model_style import ModelStyle
-from model_handler.utils import (
+from bfcl.model_handler.handler import BaseHandler
+from bfcl.model_handler.model_style import ModelStyle
+from bfcl.model_handler.utils import (
     ast_parse,
     augment_prompt_by_languge,
     language_specific_pre_processing,
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/utils.py b/berkeley-function-call-leaderboard/bfcl/model_handler/utils.py
index 485171e48..d5ecd5d26 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/utils.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/utils.py
@@ -1,8 +1,8 @@
 import re, ast, builtins, ast, json
-from model_handler.model_style import ModelStyle
-from model_handler.java_parser import parse_java_function_call
-from model_handler.js_parser import parse_javascript_function_call
-from model_handler.constant import GORILLA_TO_OPENAPI, USE_COHERE_OPTIMIZATION
+from bfcl.model_handler.model_style import ModelStyle
+from bfcl.model_handler.java_parser import parse_java_function_call
+from bfcl.model_handler.js_parser import parse_javascript_function_call
+from bfcl.model_handler.constant import GORILLA_TO_OPENAPI, USE_COHERE_OPTIMIZATION
 
 
 def _cast_to_openai_type(properties, mapping, test_category):
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/xlam_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/xlam_handler.py
index 9d0ab1e9f..60869a8ab 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/xlam_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/xlam_handler.py
@@ -1,7 +1,7 @@
 import json
 
-from model_handler.oss_handler import OSSHandler
-from model_handler.model_style import ModelStyle
+from bfcl.model_handler.oss_handler import OSSHandler
+from bfcl.model_handler.model_style import ModelStyle
 
 SYSTEM_PROMPT ="""You are an AI assistant for function calling. 
 For politically sensitive questions, security and privacy issues, 
diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/yi_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/yi_handler.py
index 8e8967cf6..f5ca2aea5 100644
--- a/berkeley-function-call-leaderboard/bfcl/model_handler/yi_handler.py
+++ b/berkeley-function-call-leaderboard/bfcl/model_handler/yi_handler.py
@@ -1,12 +1,12 @@
-from model_handler.handler import BaseHandler
-from model_handler.model_style import ModelStyle
-from model_handler.utils import (
+from bfcl.model_handler.handler import BaseHandler
+from bfcl.model_handler.model_style import ModelStyle
+from bfcl.model_handler.utils import (
     convert_to_tool,
     convert_to_function_call,
     augment_prompt_by_languge,
     language_specific_pre_processing,
 )
-from model_handler.constant import GORILLA_TO_OPENAPI
+from bfcl.model_handler.constant import GORILLA_TO_OPENAPI
 from openai import OpenAI
 import os, time, json
 

From 50a1a7adca214d995d6a4f8d3702de249bf6447b Mon Sep 17 00:00:00 2001
From: Huanzhi Mao <huanzhimao@gmail.com>
Date: Thu, 1 Aug 2024 15:55:01 -0700
Subject: [PATCH 08/18] bump python version to 3.10

---
 berkeley-function-call-leaderboard/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/berkeley-function-call-leaderboard/setup.py b/berkeley-function-call-leaderboard/setup.py
index 863ef21b5..91f78e42f 100644
--- a/berkeley-function-call-leaderboard/setup.py
+++ b/berkeley-function-call-leaderboard/setup.py
@@ -8,7 +8,7 @@
     author_email="sky@cs.berkeley.edu",
     long_description=open("README.md").read(),
     long_description_content_type="text/markdown",
-    python_requires=">=3.9",
+    python_requires=">=3.10",
     license="Apache 2.0",
     packages=find_packages(include=["bfcl*"]),
     install_requires=[

From 510ed52c24efec7ae78021d0d75942be0851a8e0 Mon Sep 17 00:00:00 2001
From: Huanzhi Mao <huanzhimao@gmail.com>
Date: Thu, 1 Aug 2024 16:07:26 -0700
Subject: [PATCH 09/18] update setup instruction

---
 berkeley-function-call-leaderboard/README.md | 33 ++++++++++++++++++--
 1 file changed, 30 insertions(+), 3 deletions(-)

diff --git a/berkeley-function-call-leaderboard/README.md b/berkeley-function-call-leaderboard/README.md
index e086d7303..978d17b3b 100644
--- a/berkeley-function-call-leaderboard/README.md
+++ b/berkeley-function-call-leaderboard/README.md
@@ -12,15 +12,42 @@ We introduce the Berkeley Function Leaderboard (BFCL), the **first comprehensive
 Read more about the technical details and interesting insights in our [blog post](https://gorilla.cs.berkeley.edu/blogs/8_berkeley_function_calling_leaderboard.html)!
 
 ![image](./architecture_diagram.png)
+
+
 ### Install Dependencies
 
 ```bash
+# Create a new Conda environment with Python 3.10
 conda create -n BFCL python=3.10
+
+# Activate the new environment
 conda activate BFCL
-pip install -r requirements.txt # Inside gorilla/berkeley-function-call-leaderboard
-pip install vllm==0.5.0 # If you have vLLM supported GPU(s) and want to run our evaluation data against self-hosted OSS models.
+
+# Clone the Gorilla repository
+git clone https://github.com/ShishirPatil/gorilla.git
+
+# Change directory to the berkeley-function-call-leaderboard
+cd gorilla/berkeley-function-call-leaderboard
+
+# Install the package in editable mode
+pip install -e .
 ```
 
+### Installing Extra Dependencies
+
+To install extra dependencies, use the following command:
+
+```bash
+pip install -e ".[NAME]"
+```
+
+Available options for `NAME` are:
+
+| Name             | Description     |
+|------------------|-----------------|
+| `oss_eval`       | For LLM generation on self-hosted open source models. Requires GPU supported by vLLM. Only works on Linux and Windows. |
+| `proprietary_eval` | For LLM generation on proprietary models (most common, **recommended**). |
+| `all`            | Installs all dependencies, combining the above two options. Only works on Linux and Windows. |
 
 ## Execution Evaluation Data Post-processing (Can be Skipped: Necesary for Executable Test Categories)
 Add your keys into `function_credential_config.json`, so that the original placeholder values in questions, params, and answers will be reset.
@@ -171,7 +198,7 @@ In the following two sections, the optional `--test-category` parameter can be u
 
 ### Running the Checker
 
-Navigate to the `gorilla/berkeley-function-call-leaderboard/eval_checker` directory and run the `eval_runner.py` script with the desired parameters. The basic syntax is as follows:
+Navigate to the `gorilla/berkeley-function-call-leaderboard/bfcl/eval_checker` directory and run the `eval_runner.py` script with the desired parameters. The basic syntax is as follows:
 
 ```bash
 python eval_runner.py --model MODEL_NAME --test-category {TEST_CATEGORY,all,ast,executable,python,non-python}

From b849373cab4fc09fb97c3a97fa89d44f75f89fd7 Mon Sep 17 00:00:00 2001
From: Huanzhi Mao <huanzhimao@gmail.com>
Date: Thu, 1 Aug 2024 16:50:16 -0700
Subject: [PATCH 10/18] remove requirements.txt

---
 .../requirements.txt                              | 15 ---------------
 1 file changed, 15 deletions(-)
 delete mode 100644 berkeley-function-call-leaderboard/requirements.txt

diff --git a/berkeley-function-call-leaderboard/requirements.txt b/berkeley-function-call-leaderboard/requirements.txt
deleted file mode 100644
index f50bc4910..000000000
--- a/berkeley-function-call-leaderboard/requirements.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-requests==2.32.3
-tqdm==4.66.4
-tree_sitter==0.21.3
-torch==2.3.0
-ray==2.32.0
-shortuuid==1.0.13
-mistralai==0.4.2
-anthropic==0.31.1
-openai==1.35.13
-numpy==1.26.4
-cohere==5.5.8
-tree-sitter-java==0.21.0
-tree-sitter-javascript==0.21.4
-
-# vllm==0.5.0
\ No newline at end of file

From d488ba2db6cd738c343833179e9eef5fb23c3cf2 Mon Sep 17 00:00:00 2001
From: Devansh Amin <devanshamin97@gmail.com>
Date: Thu, 1 Aug 2024 20:15:02 -0400
Subject: [PATCH 11/18] Fix unicode decode error on windows

---
 berkeley-function-call-leaderboard/setup.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/berkeley-function-call-leaderboard/setup.py b/berkeley-function-call-leaderboard/setup.py
index 91f78e42f..f4f079898 100644
--- a/berkeley-function-call-leaderboard/setup.py
+++ b/berkeley-function-call-leaderboard/setup.py
@@ -1,4 +1,8 @@
 from setuptools import setup, find_packages
+import codecs
+
+with codecs.open('README.md', 'r', 'utf-8') as f:
+    long_description = f.read()
 
 setup(
     name="bfcl",
@@ -6,7 +10,7 @@
     description="Berkeley Function Calling Leaderboard (BFCL)",
     author="Sky Computing Lab",
     author_email="sky@cs.berkeley.edu",
-    long_description=open("README.md").read(),
+    long_description=long_description,
     long_description_content_type="text/markdown",
     python_requires=">=3.10",
     license="Apache 2.0",

From 7b5870a7fc2a3e61cdaa15aa1c1405c05abaa01b Mon Sep 17 00:00:00 2001
From: Huanzhi Mao <huanzhimao@gmail.com>
Date: Thu, 1 Aug 2024 23:45:52 -0700
Subject: [PATCH 12/18] remove proprietary_eval option

---
 berkeley-function-call-leaderboard/setup.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/berkeley-function-call-leaderboard/setup.py b/berkeley-function-call-leaderboard/setup.py
index f4f079898..01ae8e5a7 100644
--- a/berkeley-function-call-leaderboard/setup.py
+++ b/berkeley-function-call-leaderboard/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 import codecs
 
-with codecs.open('README.md', 'r', 'utf-8') as f:
+with codecs.open("README.md", "r", "utf-8") as f:
     long_description = f.read()
 
 setup(
@@ -27,15 +27,12 @@
         "tree-sitter-java==0.21.0",
         "tree-sitter-javascript==0.21.4",
         "openai==1.35.13",
+        "mistralai==0.4.2",
+        "anthropic==0.31.1",
+        "cohere==5.5.8",
     ],
     extras_require={
         "oss_eval": ["vllm==0.5.0"],
-        "proprietary_eval": [
-            "mistralai==0.4.2",
-            "anthropic==0.31.1",
-            "cohere==5.5.8",
-        ],
-        "all": ["bfcl[oss_eval]", "bfcl[proprietary_eval]"],
     },
     project_urls={
         "Repository": "https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard",

From 16b379f234e43697233c27837f63b5b67495744c Mon Sep 17 00:00:00 2001
From: Huanzhi Mao <huanzhimao@gmail.com>
Date: Thu, 1 Aug 2024 23:48:45 -0700
Subject: [PATCH 13/18] update README accordingly

---
 berkeley-function-call-leaderboard/README.md | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/berkeley-function-call-leaderboard/README.md b/berkeley-function-call-leaderboard/README.md
index 978d17b3b..233bfb14f 100644
--- a/berkeley-function-call-leaderboard/README.md
+++ b/berkeley-function-call-leaderboard/README.md
@@ -33,21 +33,13 @@ cd gorilla/berkeley-function-call-leaderboard
 pip install -e .
 ```
 
-### Installing Extra Dependencies
-
-To install extra dependencies, use the following command:
+### Installing Extra Dependencies for Self-Hosted Open Source Models
 
+To do LLM generation on self-hosted open source models, you need to run the following command to install the extra dependencies.
 ```bash
-pip install -e ".[NAME]"
+pip install -e ". oss_eval"
 ```
-
-Available options for `NAME` are:
-
-| Name             | Description     |
-|------------------|-----------------|
-| `oss_eval`       | For LLM generation on self-hosted open source models. Requires GPU supported by vLLM. Only works on Linux and Windows. |
-| `proprietary_eval` | For LLM generation on proprietary models (most common, **recommended**). |
-| `all`            | Installs all dependencies, combining the above two options. Only works on Linux and Windows. |
+Note that this requires GPU supported by vLLM and it can only be installed on Linux and Windows (not Mac).
 
 ## Execution Evaluation Data Post-processing (Can be Skipped: Necesary for Executable Test Categories)
 Add your keys into `function_credential_config.json`, so that the original placeholder values in questions, params, and answers will be reset.

From bbad5aa71dc783be0c91c763e4e6464b0df4abe9 Mon Sep 17 00:00:00 2001
From: Huanzhi Mao <huanzhimao@gmail.com>
Date: Thu, 1 Aug 2024 23:50:05 -0700
Subject: [PATCH 14/18] fix typo

---
 berkeley-function-call-leaderboard/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/berkeley-function-call-leaderboard/README.md b/berkeley-function-call-leaderboard/README.md
index 233bfb14f..dc57e2a87 100644
--- a/berkeley-function-call-leaderboard/README.md
+++ b/berkeley-function-call-leaderboard/README.md
@@ -37,7 +37,7 @@ pip install -e .
 
 To do LLM generation on self-hosted open source models, you need to run the following command to install the extra dependencies.
 ```bash
-pip install -e ". oss_eval"
+pip install -e .[oss_eval]
 ```
 Note that this requires GPU supported by vLLM and it can only be installed on Linux and Windows (not Mac).
 

From 4ab85cbb29208e72641de4776398d9f79668c391 Mon Sep 17 00:00:00 2001
From: Huanzhi Mao <huanzhimao@gmail.com>
Date: Thu, 1 Aug 2024 23:59:07 -0700
Subject: [PATCH 15/18] sort test categories

---
 berkeley-function-call-leaderboard/openfunctions_evaluation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/berkeley-function-call-leaderboard/openfunctions_evaluation.py b/berkeley-function-call-leaderboard/openfunctions_evaluation.py
index 82099f379..a3679d020 100644
--- a/berkeley-function-call-leaderboard/openfunctions_evaluation.py
+++ b/berkeley-function-call-leaderboard/openfunctions_evaluation.py
@@ -59,7 +59,7 @@ def parse_test_category_argument(test_category_args):
             test_name_total.add(test_category)
             test_filename_total.add(TEST_FILE_MAPPING[test_category])
 
-    return list(test_name_total), list(test_filename_total)
+    return sorted(list(test_name_total)), sorted(list(test_filename_total))
 
 
 def collect_test_cases(test_filename_total, model_name):

From 3e0953229a452edd6e8f888f56afde2f7631257f Mon Sep 17 00:00:00 2001
From: Devansh Amin <devanshamin97@gmail.com>
Date: Wed, 7 Aug 2024 18:41:07 -0400
Subject: [PATCH 16/18] Move out dependencies to a separate file

---
 .../requirements.txt                          | 14 +++++++++++++
 berkeley-function-call-leaderboard/setup.py   | 21 +++++--------------
 2 files changed, 19 insertions(+), 16 deletions(-)
 create mode 100644 berkeley-function-call-leaderboard/requirements.txt

diff --git a/berkeley-function-call-leaderboard/requirements.txt b/berkeley-function-call-leaderboard/requirements.txt
new file mode 100644
index 000000000..c30d4675f
--- /dev/null
+++ b/berkeley-function-call-leaderboard/requirements.txt
@@ -0,0 +1,14 @@
+requests
+tqdm
+numpy==1.26.4
+pandas
+huggingface_hub
+pydantic>=2.8.2
+python-dotenv>=1.0.1
+tree_sitter==0.21.3
+tree-sitter-java==0.21.0
+tree-sitter-javascript==0.21.4
+openai==1.35.13
+mistralai==0.4.2
+anthropic==0.31.1
+cohere==5.5.8
diff --git a/berkeley-function-call-leaderboard/setup.py b/berkeley-function-call-leaderboard/setup.py
index 01ae8e5a7..bde688e52 100644
--- a/berkeley-function-call-leaderboard/setup.py
+++ b/berkeley-function-call-leaderboard/setup.py
@@ -1,6 +1,10 @@
 from setuptools import setup, find_packages
 import codecs
 
+def read_requirements(filename):
+    with open(filename, 'r') as f:
+        return f.read().splitlines()
+
 with codecs.open("README.md", "r", "utf-8") as f:
     long_description = f.read()
 
@@ -15,22 +19,7 @@
     python_requires=">=3.10",
     license="Apache 2.0",
     packages=find_packages(include=["bfcl*"]),
-    install_requires=[
-        "requests",
-        "tqdm",
-        "numpy==1.26.4",
-        "pandas",
-        "huggingface_hub",
-        "pydantic>=2.8.2",
-        "python-dotenv>=1.0.1",
-        "tree_sitter==0.21.3",
-        "tree-sitter-java==0.21.0",
-        "tree-sitter-javascript==0.21.4",
-        "openai==1.35.13",
-        "mistralai==0.4.2",
-        "anthropic==0.31.1",
-        "cohere==5.5.8",
-    ],
+    install_requires=read_requirements("requirements.txt"),
     extras_require={
         "oss_eval": ["vllm==0.5.0"],
     },

From 9843d6300a365ff8f92daa18e895ca88f54891e8 Mon Sep 17 00:00:00 2001
From: Devansh Amin <devanshamin97@gmail.com>
Date: Wed, 7 Aug 2024 18:41:48 -0400
Subject: [PATCH 17/18] Remove pyproject file

---
 berkeley-function-call-leaderboard/pyproject.toml | 3 ---
 1 file changed, 3 deletions(-)
 delete mode 100644 berkeley-function-call-leaderboard/pyproject.toml

diff --git a/berkeley-function-call-leaderboard/pyproject.toml b/berkeley-function-call-leaderboard/pyproject.toml
deleted file mode 100644
index 2f2101195..000000000
--- a/berkeley-function-call-leaderboard/pyproject.toml
+++ /dev/null
@@ -1,3 +0,0 @@
-[build-system]
-requires = ["setuptools>=40.8.0", "wheel"]
-build-backend = "setuptools.build_meta"

From e87e78bbd89742dd504dfa97079f2f73f947adce Mon Sep 17 00:00:00 2001
From: Devansh Amin <devanshamin97@gmail.com>
Date: Wed, 7 Aug 2024 21:33:00 -0400
Subject: [PATCH 18/18] Revert back to pyproject for editable pip installation

---
 .../pyproject.toml                            | 39 +++++++++++++++++++
 .../requirements.txt                          | 14 -------
 berkeley-function-call-leaderboard/setup.py   | 29 --------------
 3 files changed, 39 insertions(+), 43 deletions(-)
 create mode 100644 berkeley-function-call-leaderboard/pyproject.toml
 delete mode 100644 berkeley-function-call-leaderboard/requirements.txt
 delete mode 100644 berkeley-function-call-leaderboard/setup.py

diff --git a/berkeley-function-call-leaderboard/pyproject.toml b/berkeley-function-call-leaderboard/pyproject.toml
new file mode 100644
index 000000000..922acfd7e
--- /dev/null
+++ b/berkeley-function-call-leaderboard/pyproject.toml
@@ -0,0 +1,39 @@
+[build-system]
+requires = ["setuptools>=40.8.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "bfcl"
+version = "0.1.0"
+description = "Berkeley Function Calling Leaderboard (BFCL)"
+authors = [
+    {name="Sky Computing Lab", email="sky@cs.berkeley.edu"}
+]
+readme = "README.md"
+requires-python = ">=3.10"
+license = { "text" = "Apache 2.0" }
+dependencies = [
+    "requests",
+    "tqdm",
+    "numpy==1.26.4",
+    "pandas",
+    "huggingface_hub",
+    "pydantic>=2.8.2",
+    "python-dotenv>=1.0.1",
+    "tree_sitter==0.21.3",
+    "tree-sitter-java==0.21.0",
+    "tree-sitter-javascript==0.21.4",
+    "openai==1.35.13",
+    "mistralai==0.4.2",
+    "anthropic==0.31.1",
+    "cohere==5.5.8",
+]
+
+[tool.setuptools.packages.find]
+include = ["bfcl*"]
+
+[project.urls]
+Repository = "https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard"
+
+[project.optional-dependencies]
+oss_eval = ["vllm==0.5.0"]
diff --git a/berkeley-function-call-leaderboard/requirements.txt b/berkeley-function-call-leaderboard/requirements.txt
deleted file mode 100644
index c30d4675f..000000000
--- a/berkeley-function-call-leaderboard/requirements.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-requests
-tqdm
-numpy==1.26.4
-pandas
-huggingface_hub
-pydantic>=2.8.2
-python-dotenv>=1.0.1
-tree_sitter==0.21.3
-tree-sitter-java==0.21.0
-tree-sitter-javascript==0.21.4
-openai==1.35.13
-mistralai==0.4.2
-anthropic==0.31.1
-cohere==5.5.8
diff --git a/berkeley-function-call-leaderboard/setup.py b/berkeley-function-call-leaderboard/setup.py
deleted file mode 100644
index bde688e52..000000000
--- a/berkeley-function-call-leaderboard/setup.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from setuptools import setup, find_packages
-import codecs
-
-def read_requirements(filename):
-    with open(filename, 'r') as f:
-        return f.read().splitlines()
-
-with codecs.open("README.md", "r", "utf-8") as f:
-    long_description = f.read()
-
-setup(
-    name="bfcl",
-    version="0.1.0",
-    description="Berkeley Function Calling Leaderboard (BFCL)",
-    author="Sky Computing Lab",
-    author_email="sky@cs.berkeley.edu",
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-    python_requires=">=3.10",
-    license="Apache 2.0",
-    packages=find_packages(include=["bfcl*"]),
-    install_requires=read_requirements("requirements.txt"),
-    extras_require={
-        "oss_eval": ["vllm==0.5.0"],
-    },
-    project_urls={
-        "Repository": "https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard",
-    },
-)