Remove dataset download twice

Signed-off-by: Andrey Velichkevich <[email protected]>
andreyvelich · Mar 15, 2024 · 6717f93 · 6717f93
1 parent f3fb861
commit 6717f93
Showing 1 changed file with 1 addition and 6 deletions.
diff --git a/sdk/python/kubeflow/storage_initializer/hugging_face.py b/sdk/python/kubeflow/storage_initializer/hugging_face.py
@@ -102,11 +102,6 @@ def download_dataset(self):
         if self.config.access_token:
             huggingface_hub.login(self.config.access_token)
 
-        load_dataset(self.config.repo_id, cache_dir=VOLUME_PATH_DATASET)
-
         # Load dataset and save to disk.
-        dataset = load_dataset(
-            self.config.repo_id,
-            split=self.config.split,
-        )
+        dataset = load_dataset(self.config.repo_id, split=self.config.split)
         dataset.save_to_disk(VOLUME_PATH_DATASET)