added a warning

allenai · Jan 20, 2024 · ce71dcf · ce71dcf
1 parent 62f4008
commit ce71dcf
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/python/dolma/cli/tokenizer.py b/python/dolma/cli/tokenizer.py
@@ -49,6 +49,13 @@ def __post__init__(self):
             logger.warning("No pad token ID provided; using EOS token ID.")
             self.pad_token_id = self.eos_token_id
 
+        if self.segment_before_tokenization:
+            logger.warning(
+                "EXPERIMENTAL FEATURE: segmenting before tokenization is enabled. "
+                "This option has only been tested with Llama and GPT-NeoX tokenizers. "
+                "USE AT YOUR OWN RISK."
+            )
+
     @classmethod
     def deprecated_init(cls, tokenizer_name_or_path: str) -> "TokenizerConfig":
         logger = get_logger(__file__)