From e002b8ac974fc2c0f8f929ed69fc08858189a51d Mon Sep 17 00:00:00 2001
From: Yuki Iwai <yuki.iwai.tz@gmail.com>
Date: Mon, 19 Jun 2023 02:30:38 +0900
Subject: [PATCH] Add check pods are not scheduled when testing gang-scheduler
 integrations in e2e (#1835)

Signed-off-by: Yuki Iwai <yuki.iwai.tz@gmail.com>
---
 sdk/python/test/e2e/utils.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/sdk/python/test/e2e/utils.py b/sdk/python/test/e2e/utils.py
index cd9223ba46..32994be79c 100644
--- a/sdk/python/test/e2e/utils.py
+++ b/sdk/python/test/e2e/utils.py
@@ -1,4 +1,5 @@
 import logging
+import time
 
 from kubeflow.training import TrainingClient
 from kubeflow.training.constants import constants
@@ -17,13 +18,19 @@ def verify_unschedulable_job_e2e(
     logging.info(f"\n\n\n{job_kind} is creating")
     client.wait_for_job_conditions(name, namespace, job_kind, {constants.JOB_CONDITION_CREATED})
 
-    # Job should have a Created condition.
-    if not client.is_job_created(name, namespace, job_kind):
-        raise Exception(f"{job_kind} should be in Created condition")
+    logging.info("Checking 3 times that pods are not scheduled")
+    for num in range(3):
+        logging.info(f"Number of attempts: {int(num)+1}/3")
+        # Job should have a Created condition.
+        if not client.is_job_created(name, namespace, job_kind):
+            raise Exception(f"{job_kind} should be in Created condition")
 
-    # Job shouldn't have a Running condition.
-    if client.is_job_running(name, namespace, job_kind):
-        raise Exception(f"{job_kind} shouldn't be in Running condition")
+        # Job shouldn't have a Running condition.
+        if client.is_job_running(name, namespace, job_kind):
+            raise Exception(f"{job_kind} shouldn't be in Running condition")
+
+        logging.info("Sleeping 5 seconds...")
+        time.sleep(5)
 
 
 def verify_job_e2e(
@@ -68,8 +75,7 @@ def verify_job_e2e(
 def get_pod_spec_scheduler_name(gang_scheduler_name: str) -> str:
     if gang_scheduler_name == TEST_GANG_SCHEDULER_NAME_SCHEDULER_PLUGINS:
         return DEFAULT_SCHEDULER_PLUGINS_NAME
-    # TODO (tenzen-y): Implement E2E tests using volcano.
     elif gang_scheduler_name == TEST_GANG_SCHEDULER_NAME_VOLCANO:
-        return ""
+        return TEST_GANG_SCHEDULER_NAME_VOLCANO
 
     return ""