Merge pull request #12 from JenniferOH/fix_tokenizer_init

transformer 4.34.0 버전부터 tokenizer init 단계에서 vocab을 참조합니다.

kobert_transformers/tokenization_kobert.py

@@ -82,14 +82,6 @@ def __init__(
         mask_token="[MASK]",
         **kwargs,
     ):
-        super().__init__(
-            unk_token=unk_token,
-            sep_token=sep_token,
-            pad_token=pad_token,
-            cls_token=cls_token,
-            mask_token=mask_token,
-            **kwargs,
-        )
 
         # Build vocab
         self.token2idx = dict()
@@ -117,6 +109,15 @@ def __init__(
         self.sp_model = spm.SentencePieceProcessor()
         self.sp_model.Load(vocab_file)
 
+        super().__init__(
+            unk_token=unk_token,
+            sep_token=sep_token,
+            pad_token=pad_token,
+            cls_token=cls_token,
+            mask_token=mask_token,
+            **kwargs,
+        )
+
     @property
     def vocab_size(self):
         return len(self.idx2token)