From 4635a927fa1148007c71627c0a5003fa24df7bca Mon Sep 17 00:00:00 2001 From: lvliang-intel Date: Thu, 7 Nov 2024 17:39:34 +0800 Subject: [PATCH] Make embedding run on CPU for aligning with Gaudi performance benchmark (#1057) Signed-off-by: lvliang-intel Co-authored-by: chen, suyue --- ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml | 12 ++---------- .../intel/hpu/gaudi/compose_guardrails.yaml | 12 ++---------- .../docker_compose/intel/hpu/gaudi/compose_vllm.yaml | 12 ++---------- .../intel/hpu/gaudi/compose_vllm_ray.yaml | 12 ++---------- .../intel/hpu/gaudi/compose_without_rerank.yaml | 12 ++---------- 5 files changed, 10 insertions(+), 50 deletions(-) diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml index e34f072b5..a47575b31 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml @@ -26,25 +26,17 @@ services: TEI_ENDPOINT: http://tei-embedding-service:80 HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} tei-embedding-service: - image: ghcr.io/huggingface/tei-gaudi:latest + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 container_name: tei-embedding-gaudi-server ports: - "8090:80" volumes: - "./data:/data" - runtime: habana - cap_add: - - SYS_NICE - ipc: host + shm_size: 1g environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - HF_HUB_DISABLE_PROGRESS_BARS: 1 - HF_HUB_ENABLE_HF_TRANSFER: 0 - HABANA_VISIBLE_DEVICES: all - OMPI_MCA_btl_vader_single_copy_mechanism: none - MAX_WARMUP_SEQUENCE_LENGTH: 512 command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate retriever: image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest} diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_guardrails.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_guardrails.yaml index 7b9d391fe..570d689c2 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_guardrails.yaml +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_guardrails.yaml @@ -65,25 +65,17 @@ services: HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} restart: unless-stopped tei-embedding-service: - image: ghcr.io/huggingface/tei-gaudi:latest + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 container_name: tei-embedding-gaudi-server ports: - "8090:80" volumes: - "./data:/data" - runtime: habana - cap_add: - - SYS_NICE - ipc: host + shm_size: 1g environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - HF_HUB_DISABLE_PROGRESS_BARS: 1 - HF_HUB_ENABLE_HF_TRANSFER: 0 - HABANA_VISIBLE_DEVICES: all - OMPI_MCA_btl_vader_single_copy_mechanism: none - MAX_WARMUP_SEQUENCE_LENGTH: 512 command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate retriever: image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest} diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_vllm.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_vllm.yaml index acd01bb2a..0f42a3fc6 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_vllm.yaml +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_vllm.yaml @@ -26,25 +26,17 @@ services: TEI_ENDPOINT: http://tei-embedding-service:80 HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} tei-embedding-service: - image: ghcr.io/huggingface/tei-gaudi:latest + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 container_name: tei-embedding-gaudi-server ports: - "8090:80" volumes: - "./data:/data" - runtime: habana - cap_add: - - SYS_NICE - ipc: host + shm_size: 1g environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - HF_HUB_DISABLE_PROGRESS_BARS: 1 - HF_HUB_ENABLE_HF_TRANSFER: 0 - HABANA_VISIBLE_DEVICES: all - OMPI_MCA_btl_vader_single_copy_mechanism: none - MAX_WARMUP_SEQUENCE_LENGTH: 512 command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate retriever: image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest} diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_vllm_ray.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_vllm_ray.yaml index a1019c9ac..1f067e77e 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_vllm_ray.yaml +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_vllm_ray.yaml @@ -26,25 +26,17 @@ services: TEI_ENDPOINT: http://tei-embedding-service:80 HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} tei-embedding-service: - image: ghcr.io/huggingface/tei-gaudi:latest + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 container_name: tei-embedding-gaudi-server ports: - "8090:80" volumes: - "./data:/data" - runtime: habana - cap_add: - - SYS_NICE - ipc: host + shm_size: 1g environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - HF_HUB_DISABLE_PROGRESS_BARS: 1 - HF_HUB_ENABLE_HF_TRANSFER: 0 - HABANA_VISIBLE_DEVICES: all - OMPI_MCA_btl_vader_single_copy_mechanism: none - MAX_WARMUP_SEQUENCE_LENGTH: 512 command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate retriever: image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest} diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_without_rerank.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_without_rerank.yaml index 7c2323157..9117a13a6 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_without_rerank.yaml +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_without_rerank.yaml @@ -26,25 +26,17 @@ services: TEI_ENDPOINT: http://tei-embedding-service:80 HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} tei-embedding-service: - image: ghcr.io/huggingface/tei-gaudi:latest + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 container_name: tei-embedding-gaudi-server ports: - "8090:80" volumes: - "./data:/data" - runtime: habana - cap_add: - - SYS_NICE - ipc: host + shm_size: 1g environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - HF_HUB_DISABLE_PROGRESS_BARS: 1 - HF_HUB_ENABLE_HF_TRANSFER: 0 - HABANA_VISIBLE_DEVICES: all - OMPI_MCA_btl_vader_single_copy_mechanism: none - MAX_WARMUP_SEQUENCE_LENGTH: 512 command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate retriever: image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest}