From 70366edb7cef59ff27bdde888b747a897b1f0a2c Mon Sep 17 00:00:00 2001 From: sychen Date: Mon, 30 Oct 2023 14:21:06 +0800 Subject: [PATCH] [CELEBORN-1096] Avoid initializing SortShuffleManager when stop ### What changes were proposed in this pull request? ### Why are the changes needed? If there is no fallback to the Spark SortShuffleManager, it will be initialized once when stopping, and then stopped again. This is not necessary. ### Does this PR introduce _any_ user-facing change? ### How was this patch tested? Closes #2052 from cxzl25/CELEBORN-1096. Authored-by: sychen Signed-off-by: Fu Chen --- .../apache/spark/shuffle/celeborn/SparkShuffleManager.java | 5 +++-- .../apache/spark/shuffle/celeborn/SparkShuffleManager.java | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java index 74e983c2a7..071810ad29 100644 --- a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java +++ b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java @@ -167,8 +167,9 @@ public void stop() { if (lifecycleManager != null) { lifecycleManager.stop(); } - if (sortShuffleManager() != null) { - sortShuffleManager().stop(); + if (_sortShuffleManager != null) { + _sortShuffleManager.stop(); + _sortShuffleManager = null; } } diff --git a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java index c0d3bd2864..aa1c1a2c25 100644 --- a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java +++ b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java @@ -197,8 +197,8 @@ public void stop() { lifecycleManager.stop(); lifecycleManager = null; } - if (sortShuffleManager() != null) { - sortShuffleManager().stop(); + if (_sortShuffleManager != null) { + _sortShuffleManager.stop(); _sortShuffleManager = null; } }