From be494053a0e10a006bdee003b7e61384d723884e Mon Sep 17 00:00:00 2001 From: Paddy Xu Date: Thu, 10 Oct 2024 13:27:20 +0200 Subject: [PATCH] why so slow try 2 --- .../org/apache/spark/sql/artifact/ArtifactManager.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala index 4f64005aa3f12..e80ad1404d3ca 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala @@ -36,7 +36,7 @@ import org.apache.spark.sql.{Artifact, SparkSession} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.util.ArtifactUtils import org.apache.spark.storage.{BlockManager, CacheId, StorageLevel} -import org.apache.spark.util.{ChildFirstURLClassLoader, JoinClassLoader, StubClassLoader, Utils} +import org.apache.spark.util.{ChildFirstURLClassLoader, StubClassLoader, Utils} /** * This class handles the storage of artifacts as well as preparing the artifacts for use. @@ -281,8 +281,7 @@ class ArtifactManager(session: SparkSession) extends Logging { val urls = getAddedJars :+ classDir.toUri.toURL val prefixes = SparkEnv.get.conf.get(CONNECT_SCALA_UDF_STUB_PREFIXES) val userClasspathFirst = SparkEnv.get.conf.get(EXECUTOR_USER_CLASS_PATH_FIRST) - val fallbackClassLoader = // jarClassLoader -> contextClassLoader - new JoinClassLoader(session.sharedState.jarClassLoader, Utils.getContextOrSparkClassLoader) + val fallbackClassLoader = Utils.getContextOrSparkClassLoader val loader = if (prefixes.nonEmpty) { // Two things you need to know about classloader for all of this to make sense: // 1. A classloader needs to be able to fully define a class.