From 24ba4d41eaffb1bff5e9ec1eaec317ae55f6cdd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Kot=C5=82owski?= Date: Mon, 4 Nov 2024 09:43:06 +0100 Subject: [PATCH] [CI] Add Llama2 to torch compile tests (#446) --- .../lm-eval-harness/configs/Llama-2-7B-hf.yaml | 14 ++++++++++++++ .jenkins/lm-eval-harness/configs/models-llama2.txt | 1 + .jenkins/test_config_t_compile.yaml | 12 ++++++++++++ 3 files changed, 27 insertions(+) create mode 100644 .jenkins/lm-eval-harness/configs/Llama-2-7B-hf.yaml create mode 100644 .jenkins/lm-eval-harness/configs/models-llama2.txt diff --git a/.jenkins/lm-eval-harness/configs/Llama-2-7B-hf.yaml b/.jenkins/lm-eval-harness/configs/Llama-2-7B-hf.yaml new file mode 100644 index 0000000000000..da048ba19305f --- /dev/null +++ b/.jenkins/lm-eval-harness/configs/Llama-2-7B-hf.yaml @@ -0,0 +1,14 @@ +# These scores were chosen to place within 6% range of values achieved using vLLM on HPU: +# 0.148 - 0.164 +# where on https://www.llama.com/llama2/: 0.146 is given +model_name: "/mnt/weka/data/pytorch/llama2/Llama-2-7b-hf" +tasks: +- name: "gsm8k" + metrics: + - name: "exact_match,strict-match" + value: 0.155 + - name: "exact_match,flexible-extract" + value: 0.155 +limit: 250 +num_fewshot: 5 +dtype: "bfloat16" \ No newline at end of file diff --git a/.jenkins/lm-eval-harness/configs/models-llama2.txt b/.jenkins/lm-eval-harness/configs/models-llama2.txt new file mode 100644 index 0000000000000..7ae5af4cce4d3 --- /dev/null +++ b/.jenkins/lm-eval-harness/configs/models-llama2.txt @@ -0,0 +1 @@ +Llama-2-7B-hf.yaml \ No newline at end of file diff --git a/.jenkins/test_config_t_compile.yaml b/.jenkins/test_config_t_compile.yaml index 58fcb45a7edfb..da20c3486aa86 100644 --- a/.jenkins/test_config_t_compile.yaml +++ b/.jenkins/test_config_t_compile.yaml @@ -14,3 +14,15 @@ stages: - name: gsm8k_small_g2_tp2_tc flavor: g2.s command: cd .jenkins/lm-eval-harness && PT_HPU_LAZY_MODE=0 bash run-tests.sh -c configs/models-small.txt -t 2 + - name: gsm8k_llama2_g3_tp1_tc + flavor: g3 + command: cd .jenkins/lm-eval-harness && PT_HPU_LAZY_MODE=0 bash run-tests.sh -c configs/models-llama2.txt -t 1 + - name: gsm8k_llama2_g3_tp2_tc + flavor: g3.s + command: cd .jenkins/lm-eval-harness && PT_HPU_LAZY_MODE=0 bash run-tests.sh -c configs/models-llama2.txt -t 2 + - name: gsm8k_lama2_g2_tp1_tc + flavor: g2 + command: cd .jenkins/lm-eval-harness && PT_HPU_LAZY_MODE=0 bash run-tests.sh -c configs/models-llama2.txt -t 1 + - name: gsm8k_lama2_g2_tp2_tc + flavor: g2.s + command: cd .jenkins/lm-eval-harness && PT_HPU_LAZY_MODE=0 bash run-tests.sh -c configs/models-llama2.txt -t 2 \ No newline at end of file