From b8863892022625b6032dda15dcaa162387f37813 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Tue, 3 Sep 2024 19:51:12 +0530 Subject: [PATCH 1/3] delete the stateful set of worker pool once the pool is deleted --- packages/syft/src/syft/service/worker/worker_pool_service.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/syft/src/syft/service/worker/worker_pool_service.py b/packages/syft/src/syft/service/worker/worker_pool_service.py index fd8d61841bd..dc4885e5077 100644 --- a/packages/syft/src/syft/service/worker/worker_pool_service.py +++ b/packages/syft/src/syft/service/worker/worker_pool_service.py @@ -623,7 +623,10 @@ def delete( ) if IN_KUBERNETES: + # Scale the workers to zero self.scale(context=context, number=0, pool_id=uid) + runner = KubernetesRunner() + runner.delete_pool(pool_name=worker_pool.name) else: workers = ( worker.resolve_with_context(context=context).unwrap() From b5f5c3fa1595485cbc89333725962eddb13aa31c Mon Sep 17 00:00:00 2001 From: IonesioJunior Date: Tue, 3 Sep 2024 14:03:48 -0300 Subject: [PATCH 2/3] Increate create_pool_timeout --- packages/syft/src/syft/custom_worker/runner_k8s.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/custom_worker/runner_k8s.py b/packages/syft/src/syft/custom_worker/runner_k8s.py index 6aff4bbff64..2c7e0a74bdc 100644 --- a/packages/syft/src/syft/custom_worker/runner_k8s.py +++ b/packages/syft/src/syft/custom_worker/runner_k8s.py @@ -13,7 +13,7 @@ from .k8s import get_kr8s_client JSONPATH_AVAILABLE_REPLICAS = "{.status.availableReplicas}" -CREATE_POOL_TIMEOUT_SEC = 180 +CREATE_POOL_TIMEOUT_SEC = 380 SCALE_POOL_TIMEOUT_SEC = 60 From 1da40cb3b3aa378bd5e50464536445c62ec52f56 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Tue, 3 Sep 2024 17:09:43 -0400 Subject: [PATCH 3/3] add waits to delete_pool --- packages/syft/src/syft/custom_worker/runner_k8s.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/syft/src/syft/custom_worker/runner_k8s.py b/packages/syft/src/syft/custom_worker/runner_k8s.py index 2c7e0a74bdc..e320bae8e94 100644 --- a/packages/syft/src/syft/custom_worker/runner_k8s.py +++ b/packages/syft/src/syft/custom_worker/runner_k8s.py @@ -96,9 +96,11 @@ def delete_pool(self, pool_name: str) -> bool: selector = {"app.kubernetes.io/component": pool_name} for _set in self.client.get("statefulsets", label_selector=selector): _set.delete(propagation_policy="Foreground") + _set.wait(conditions="delete") for _secret in self.client.get("secrets", label_selector=selector): _secret.delete(propagation_policy="Foreground") + _secret.wait(conditions="delete") return True