Don't skip tokenizer init

Signed-off-by: jthomson04 <[email protected]>

Author: jthomson04

components/src/dynamo/trtllm/main.py

@@ -175,16 +175,6 @@ async def init(runtime: DistributedRuntime, config: Config):
         dynamic_batch_config=dynamic_batch_config,
     )
     modality = getattr(config, "modality", None) or "text"
-    if config.use_trtllm_tokenizer:
-        logging.info(
-            "Using TensorRT-LLM's built in tokenizer. Setting skip_tokenizer_init to False"
-        )
-        skip_tokenizer_init = False
-    else:
-        logging.info(
-            "Using dynamo's built in tokenizer. Setting skip_tokenizer_init to True"
-        )
-        skip_tokenizer_init = True
 
     arg_map = {
         "model": model_path,
@@ -193,7 +183,6 @@ async def init(runtime: DistributedRuntime, config: Config):
         "pipeline_parallel_size": config.pipeline_parallel_size,
         "moe_expert_parallel_size": config.expert_parallel_size,
         "backend": Backend.PYTORCH,
-        "skip_tokenizer_init": skip_tokenizer_init,
         "build_config": build_config,
         "kv_cache_config": kv_cache_config,
         "gpus_per_node": gpus_per_node,