File tree Expand file tree Collapse file tree
src/diffusers/pipelines/kolors Expand file tree Collapse file tree Original file line number Diff line number Diff line change 1616import os
1717import re
1818
19- from sentencepiece import SentencePieceProcessor
2019from transformers import PreTrainedTokenizer
2120from transformers .tokenization_utils_base import BatchEncoding , EncodedInput
2221from transformers .utils import PaddingStrategy
2322
23+ from ...utils import is_sentencepiece_available
24+
2425
2526class SPTokenizer :
2627 def __init__ (self , model_path : str ):
28+ if not is_sentencepiece_available ():
29+ raise ImportError (
30+ "`SPTokenizer` requires the `sentencepiece` library but it was not found in your environment. You can "
31+ "install it with `pip install sentencepiece`."
32+ )
33+ from sentencepiece import SentencePieceProcessor
34+
2735 # reload tokenizer
2836 assert os .path .isfile (model_path ), model_path
2937 self .sp_model = SentencePieceProcessor (model_file = model_path )
You can’t perform that action at this time.
0 commit comments