diff --git a/tensorrt_llm/tokenizer/tokenizer.py b/tensorrt_llm/tokenizer/tokenizer.py index 9715865c65a1..b88b6b53ea45 100644 --- a/tensorrt_llm/tokenizer/tokenizer.py +++ b/tensorrt_llm/tokenizer/tokenizer.py @@ -65,6 +65,12 @@ class TokenizerBase(PreTrainedTokenizerBase): ''' This is a protocol for the tokenizer. Users can implement their own tokenizer by inheriting this class. ''' + def __repr__(self) -> str: + # PreTrainedTokenizerBase.__repr__ reads properties (e.g. + # added_tokens_decoder) that TokenizerBase subclasses are not + # required to implement, so fall back to a class-name-only repr. + return f"{self.__class__.__name__}()" + def _reconstruct_transformers_tokenizer(inner_bytes: bytes): '''Reconstruct a TransformersTokenizer from cloudpickle-serialized bytes.''' diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 398504421c9d..d537907c7257 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -377,7 +377,6 @@ unittest/executor/test_rpc_proxy.py SKIP (https://nvbugs/5605741) unittest/executor/test_rpc_worker.py SKIP (https://nvbugs/5605741) unittest/llmapi/test_additional_model_outputs.py -m "gpu2" SKIP (https://nvbugs/6428091) unittest/llmapi/test_llm.py::test_generate_with_detokenization_stop_words_streaming[/scratch.trt_llm_data/llm-models/gemma/gemma-3-1b-it] SKIP (https://nvbugs/6566772) -unittest/llmapi/test_llm.py::test_llm_with_customized_tokenizer SKIP (https://nvbugs/6507080) unittest/llmapi/test_llm_multi_gpu_pytorch.py -m "gpu2" SKIP (https://nvbugs/6428092) unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_llm_get_stats_pp2[False-False-True] SKIP (https://nvbugs/6432826) unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_phi3_lora_fused_modules_output_on_tp2_identical_to_tp1 SKIP (https://nvbugs/6109745)