diff --git a/tests/onnxruntime/test_modeling.py b/tests/onnxruntime/test_modeling.py
index abf508a80c..501c7dac24 100644
--- a/tests/onnxruntime/test_modeling.py
+++ b/tests/onnxruntime/test_modeling.py
@@ -1265,7 +1265,7 @@ def test_trust_remote_code(self):
         )
 
     @parameterized.expand(("", "onnx"))
-    def test_loading_with_config_in_root(self, subfolder):
+    def test_loading_with_config_not_from_subfolder(self, subfolder):
         # config.json file in the root directory and not in the subfolder
         model_id = "sentence-transformers-testing/stsb-bert-tiny-onnx"
         # hub model
@@ -1276,6 +1276,7 @@ def test_loading_with_config_in_root(self, subfolder):
             local_dir = Path(tmpdirname) / "model"
             api.snapshot_download(repo_id=model_id, local_dir=local_dir)
             ORTModelForFeatureExtraction.from_pretrained(local_dir, subfolder=subfolder, export=subfolder == "")
+            remove_directory(tmpdirname)
 
 
 class ORTModelForQuestionAnsweringIntegrationTest(ORTModelTestMixin):