remove unnecessory ignore

mosaicml · Apr 23, 2024 · f2ed1d7 · f2ed1d7
1 parent cbfec68
commit f2ed1d7
Showing 1 changed file with 1 addition and 3 deletions.
diff --git a/llmfoundry/utils/builders.py b/llmfoundry/utils/builders.py
@@ -443,9 +443,7 @@ def build_tokenizer(
             int(1e30),
         )
 
-    if not hasattr(
-            tokenizer, 'eos_token'
-    ) or tokenizer.eos_token is None:  # type: ignore (sometime's it's not none but that's ok too)
+    if not hasattr(tokenizer, 'eos_token') or tokenizer.eos_token is None:
         raise ValueError(
             f'The tokenizer {tokenizer_name} must have an eos_token.')