🐺 fix python tokenizer path

Dockerfile

@@ -1,4 +1,4 @@
-FROM nvcr.io/nvidia/tritonserver:22.07-py3
+FROM nvcr.io/nvidia/tritonserver:23.06-py3
 
 # see .dockerignore to check what is transfered
 
@@ -8,7 +8,7 @@ RUN apt-get update && \
     python3-distutils \
     python3-venv \
     python3-pip \
-    apt-get clean
+    && apt-get clean
 
 ARG UID=1000
 ARG GID=1000

demo/generative-model/gpt2.ipynb

@@ -146,7 +146,7 @@
     "import tensorrt as trt\n",
     "import torch\n",
     "from tensorrt import ICudaEngine\n",
-    "from tensorrt.tensorrt import Logger, Runtime\n",
+    "from tensorrt import Logger, Runtime\n",
     "from transformers import AutoTokenizer, BatchEncoding, GPT2LMHeadModel, AutoModelForCausalLM\n",
     "from transformers.modeling_outputs import BaseModelOutputWithPastAndCrossAttentions\n",
     "from transformer_deploy.utils.generative_model import GPTModelWrapper\n",

demo/quantization/quantization_end_to_end.ipynb

@@ -228,7 +228,7 @@
         "import torch\n",
         "import transformers\n",
         "from datasets import load_dataset, load_metric\n",
-        "from tensorrt.tensorrt import IExecutionContext, Logger, Runtime\n",
+        "from tensorrt import IExecutionContext, Logger, Runtime\n",
         "\n",
         "from transformers import (\n",
         "    AutoModelForSequenceClassification,\n",

demo/torchdynamo/benchmark.ipynb

@@ -183,7 +183,7 @@
     "\n",
     "import gc\n",
     "import tensorrt as trt\n",
-    "from tensorrt.tensorrt import ICudaEngine, Logger, Runtime\n",
+    "from tensorrt import ICudaEngine, Logger, Runtime\n",
     "import onnx\n",
     "from transformer_deploy.backends.trt_utils import build_engine, save_engine"
    ]

demo/torchdynamo/dynamo_utils.py

@@ -25,7 +25,7 @@
 from matplotlib.axes import Axes
 from matplotlib.figure import Figure
 from onnxruntime import GraphOptimizationLevel
-from tensorrt.tensorrt import Runtime
+from tensorrt import Runtime
 from torch._C._autograd import ProfilerActivity
 from torchdynamo.eval_frame import OptimizeContext
 from transformers import PreTrainedModel

docs/python.md

@@ -21,7 +21,7 @@ Now we need to convert to TensorRT:
 
 ```python
 import tensorrt as trt
-from tensorrt.tensorrt import Logger, Runtime
+from tensorrt import Logger, Runtime
 
 from transformer_deploy.backends.trt_utils import build_engine
 
@@ -50,7 +50,7 @@ Now the engine is ready, we can prepare the inference:
 
 ```python
 import torch
-from tensorrt.tensorrt import IExecutionContext
+from tensorrt import IExecutionContext
 
 from transformer_deploy.backends.trt_utils import get_binding_idxs
 

requirements_gpu.txt

@@ -1,5 +1,5 @@
 onnxruntime-gpu==1.13.1
-nvidia-tensorrt==8.4.1.5
 onnx_graphsurgeon
 polygraphy
 cupy-cuda117
+tensorrt==8.6.1

src/transformer_deploy/backends/trt_utils.py

@@ -24,7 +24,7 @@
 import tensorrt as trt
 import torch
 from tensorrt import ICudaEngine, IExecutionContext, ILayer, INetworkDefinition, Logger, Runtime
-from tensorrt.tensorrt import Builder, IBuilderConfig, IElementWiseLayer, IOptimizationProfile, IReduceLayer, OnnxParser
+from tensorrt import Builder, IBuilderConfig, IElementWiseLayer, IOptimizationProfile, IReduceLayer, OnnxParser
 
 
 @dataclass

src/transformer_deploy/convert.py

@@ -295,7 +295,7 @@ def get_pytorch_infer(model: PreTrainedModel, cuda: bool, task: str):
         logging.info("preparing TensorRT (FP16) benchmark")
         try:
             import tensorrt as trt
-            from tensorrt.tensorrt import ICudaEngine, Logger, Runtime
+            from tensorrt import ICudaEngine, Logger, Runtime
 
             from transformer_deploy.backends.trt_utils import build_engine, load_engine, save_engine
         except ImportError:

src/transformer_deploy/t5_utils/conversion_utils.py

@@ -630,7 +630,7 @@ def onnx_to_tensorrt_model(
     runtime, onnx_model_path, trt_logger, workspace_size, quantization, tensorrt_model_path, **kwargs
 ) -> Callable[[Dict[str, torch.Tensor]], Dict[str, torch.Tensor]]:
     try:
-        from tensorrt.tensorrt import ICudaEngine
+        from tensorrt import ICudaEngine
 
         from transformer_deploy.backends.trt_utils import build_engine, load_engine, save_engine
 

src/transformer_deploy/utils/python_tokenizer.py

@@ -43,7 +43,7 @@ def initialize(self, args: Dict[str, str]) -> None:
         """
         # more variables in https://github.com/triton-inference-server/python_backend/blob/main/src/python.cc
 
-        path: str = str(Path(args["model_repository"]).parent.absolute())
+        path: str = str(Path(args["model_repository"]).absolute() / args["model_version"])
         self.tokenizer = AutoTokenizer.from_pretrained(path)
         model_config = AutoConfig.from_pretrained(path)
         self.model_input_names = self.tokenizer.model_input_names