diff --git a/tests/unit/llms/test_vllm.py b/tests/unit/llms/test_vllm.py
index 96714e1ba..e0616837f 100644
--- a/tests/unit/llms/test_vllm.py
+++ b/tests/unit/llms/test_vllm.py
@@ -22,6 +22,7 @@
 from openai.types.completion_choice import CompletionChoice
 from openai.types.completion_usage import CompletionUsage
 from pydantic import BaseModel
+from transformers import AutoTokenizer
 
 from distilabel.llms import vLLM
 from distilabel.llms.vllm import ClientvLLM
@@ -101,10 +102,10 @@ class Animal(BaseModel):
 ]
 
 
-# Just a mock to avoid loading the model
 class DummyTokenizer:
     # chat_template = None
     chat_template = "template"
+    vocabulary = {"I'm": 1, "fine": 2, "thank": 3, "you": 4, "sir": 5}
 
     def __init__(self) -> None:
         pass
@@ -115,6 +116,12 @@ def apply_chat_template(self, input, **kwargs):
     def encode(self, text: str):
         return [1, 2, 3, 4, 5]
 
+    def convert_token_to_string(self, token: str) -> str:
+        return "token"
+
+    def get_vocab(self):
+        return self.vocabulary
+
 
 class TestvLLM:
     @pytest.mark.parametrize("multi_structured_output", (False, True))
@@ -148,8 +155,11 @@ def test_generate(
         expected_result: List[Dict[str, Any]],
     ) -> None:
         llm = vLLM(model="dummy")
-        llm._tokenizer = DummyTokenizer()
+        tokenizer = AutoTokenizer.from_pretrained(
+            "distilabel-internal-testing/tiny-random-mistral"
+        )
         vllm_mock = mock.MagicMock()
+        vllm_mock.get_tokenizer = mock.MagicMock(return_value=tokenizer)
         # mock the import by hacking sys.modules
         # https://stackoverflow.com/questions/60919705/how-to-mock-in-a-python-unittest-a-library-not-installed-locally
         import sys
@@ -192,8 +202,10 @@ def test_generate(
                         },
                     ],
                     {
-                        "format": "json",
-                        "schema": Character.model_json_schema(),
+                        # "format": "json",
+                        "format": "regex",
+                        "schema": r".*",
+                        # "schema": Character.model_json_schema(),
                     },
                 )
             ]