cluster status safeguards (#254)

project-codeflare · Aug 7, 2023 · f34697a · f34697a
1 parent a285ef6
commit f34697a
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 7 deletions.
diff --git a/src/codeflare_sdk/cluster/cluster.py b/src/codeflare_sdk/cluster/cluster.py
@@ -176,9 +176,15 @@ def status(
                 ready = False
                 status = CodeFlareClusterStatus.FAILED  # should deleted be separate
                 return status, ready  # exit early, no need to check ray status
-            elif appwrapper.status in [AppWrapperStatus.PENDING]:
+            elif appwrapper.status in [
+                AppWrapperStatus.PENDING,
+                AppWrapperStatus.QUEUEING,
+            ]:
                 ready = False
-                status = CodeFlareClusterStatus.QUEUED
+                if appwrapper.status == AppWrapperStatus.PENDING:
+                    status = CodeFlareClusterStatus.QUEUED
+                else:
+                    status = CodeFlareClusterStatus.QUEUEING
                 if print_to_console:
                     pretty_print.print_app_wrappers_status([appwrapper])
                 return (
@@ -561,11 +567,18 @@ def _map_to_ray_cluster(rc) -> Optional[RayCluster]:
 
 
 def _map_to_app_wrapper(aw) -> AppWrapper:
+    if "status" in aw and "canrun" in aw["status"]:
+        return AppWrapper(
+            name=aw["metadata"]["name"],
+            status=AppWrapperStatus(aw["status"]["state"].lower()),
+            can_run=aw["status"]["canrun"],
+            job_state=aw["status"]["queuejobstate"],
+        )
     return AppWrapper(
         name=aw["metadata"]["name"],
-        status=AppWrapperStatus(aw["status"]["state"].lower()),
-        can_run=aw["status"]["canrun"],
-        job_state=aw["status"]["queuejobstate"],
+        status=AppWrapperStatus("queueing"),
+        can_run=False,
+        job_state="Still adding to queue",
     )
 
 

diff --git a/src/codeflare_sdk/cluster/model.py b/src/codeflare_sdk/cluster/model.py
@@ -39,6 +39,7 @@ class AppWrapperStatus(Enum):
     Defines the possible reportable states of an AppWrapper.
     """
 
+    QUEUEING = "queueing"
     PENDING = "pending"
     RUNNING = "running"
     FAILED = "failed"
@@ -55,8 +56,9 @@ class CodeFlareClusterStatus(Enum):
     READY = 1
     STARTING = 2
     QUEUED = 3
-    FAILED = 4
-    UNKNOWN = 5
+    QUEUEING = 4
+    FAILED = 5
+    UNKNOWN = 6
 
 
 @dataclass