diff --git a/core/src/main/scala/org/apache/spark/util/Lazy.scala b/core/src/main/scala/org/apache/spark/util/Lazy.scala index d46fc6a04fff3..fa1dc207a184e 100644 --- a/core/src/main/scala/org/apache/spark/util/Lazy.scala +++ b/core/src/main/scala/org/apache/spark/util/Lazy.scala @@ -16,8 +16,6 @@ */ package org.apache.spark.util -import java.io.ObjectOutputStream - /** * Construct to lazily initialize a variable. * This may be helpful for avoiding deadlocks in certain scenarios. For example, @@ -27,7 +25,6 @@ import java.io.ObjectOutputStream * the parent object. * c) If thread 1 waits for thread 2 to join, a deadlock occurs. */ -@SerialVersionUID(7964587975756091988L) private[spark] class Lazy[T](initializer: => T) extends Serializable { private[this] lazy val value: T = initializer @@ -35,9 +32,4 @@ private[spark] class Lazy[T](initializer: => T) extends Serializable { def apply(): T = { value } - - private def writeObject(stream: ObjectOutputStream): Unit = { - this.value // ensure value is initialized - stream.defaultWriteObject() - } } diff --git a/core/src/test/scala/org/apache/spark/util/LazySuite.scala b/core/src/test/scala/org/apache/spark/util/LazySuite.scala index acc618a7557d5..19f05d2bd825a 100644 --- a/core/src/test/scala/org/apache/spark/util/LazySuite.scala +++ b/core/src/test/scala/org/apache/spark/util/LazySuite.scala @@ -16,29 +16,11 @@ */ package org.apache.spark.util -import java.io.{ByteArrayInputStream, ByteArrayOutputStream, NotSerializableException, ObjectInputStream, ObjectOutputStream} -import java.util.Base64 +import java.io.{ByteArrayOutputStream, NotSerializableException, ObjectOutputStream} import org.apache.spark.SparkFunSuite -@SerialVersionUID(2108928072825530631L) -case class LazyUsage() extends Serializable { - private val lazyString = new Lazy(LazyFunction.GetString()) - def getLazyString(): String = lazyString() -} - -object LazyFunction { - def GetString(): String = "hello world" -} - class LazySuite extends SparkFunSuite { - // This is generated with an old version (lamda is serialized): - // 1, comment out the 'writeObject' method in Lazy.scala; - // 2, print the serialized base64 string of a LazyUsage instance. - // scalastyle:off - private val base64WithLambda = "rO0ABXNyAB9vcmcuYXBhY2hlLnNwYXJrLnV0aWwuTGF6eVVzYWdlHURq6J2daQcCAAFMAApsYXp5U3RyaW5ndAAcTG9yZy9hcGFjaGUvc3BhcmsvdXRpbC9MYXp5O3hwc3IAGm9yZy5hcGFjaGUuc3BhcmsudXRpbC5MYXp5bofmkNVKElQCAANaAAhiaXRtYXAkMEwAC2luaXRpYWxpemVydAARTHNjYWxhL0Z1bmN0aW9uMDtMAAV2YWx1ZXQAEkxqYXZhL2xhbmcvT2JqZWN0O3hwAHNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzdAATW0xqYXZhL2xhbmcvT2JqZWN0O0wADmNhcHR1cmluZ0NsYXNzdAARTGphdmEvbGFuZy9DbGFzcztMABhmdW5jdGlvbmFsSW50ZXJmYWNlQ2xhc3N0ABJMamF2YS9sYW5nL1N0cmluZztMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgAKTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgAKTAAJaW1wbENsYXNzcQB+AApMAA5pbXBsTWV0aG9kTmFtZXEAfgAKTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgAKTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgAKeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHZxAH4AAHQAD3NjYWxhL0Z1bmN0aW9uMHQABWFwcGx5dAAUKClMamF2YS9sYW5nL09iamVjdDt0AB9vcmcvYXBhY2hlL3NwYXJrL3V0aWwvTGF6eVVzYWdldAAVJGFub25mdW4kbGF6eVN0cmluZyQxdAAUKClMamF2YS9sYW5nL1N0cmluZztxAH4AFHA=" - // scalastyle:on - test("Lazy val works") { var test: Option[Object] = None @@ -57,81 +39,22 @@ class LazySuite extends SparkFunSuite { assert(lazyval() == test && test.isDefined) } - test("Lazy val serialization fails if the dereferenced object is not serializable") { + test("Lazy val is serializable") { val lazyval = new Lazy({ new Object() }) - // Ensure we are not serializable after the dereference (type "Object" is not serializable) - intercept[NotSerializableException] { - val oos = new ObjectOutputStream(new ByteArrayOutputStream()) - oos.writeObject(lazyval) - } - } - - test("Lazy val serializes the value, even if dereference was never called") { - var times = 100 - val lazyval = new Lazy({ - val result = "0" * times - times = 0 - result - }) + // Ensure we are serializable before the dereference + val oos = new ObjectOutputStream(new ByteArrayOutputStream()) + oos.writeObject(lazyval) - // Ensure we serialized the value even if it was never dereferenced - val base64str = serializeToBase64(lazyval) - assert(base64str.contains("MDAw" * 32)) // MDAw is the base64 encoding of "000" - } + @SuppressWarnings(Array("never used")) + val dereferenced = lazyval() - test("Lazy val serializes the value, if dereference was called") { - var times = 100 - val lazyval = new Lazy({ - val result = "0" * times - times = 0 - result - }) - val zeros = lazyval() - assert(zeros.length == 100) - // Ensure we serialized the same value - val base64str = serializeToBase64(lazyval) - assert(base64str.contains("MDAw" * 32)) // MDAw is the base64 encoding of "000" - } - - test("Lazy val can be deserialized from lambda") { - val lazyObj = deserializeFromBase64[LazyUsage](base64WithLambda) - assert(lazyObj.getLazyString() === "hello world") - } - - test("Lazy value round trip serialization works correctly") { - val lazyval = LazyUsage() - val serialized = serializeToBase64(lazyval) - val obj = deserializeFromBase64[LazyUsage](serialized) - assert(obj.getLazyString() === lazyval.getLazyString()) - } - - test("serialVersionUID was serialized and deserialized correctly") { - val lazyval = new Lazy("") - val serialized = serializeToBase64(lazyval) - val obj = deserializeFromBase64[Lazy[String]](serialized) - val field = obj.getClass.getDeclaredField("serialVersionUID") - field.setAccessible(true) - assert(field.getLong(obj) == 7964587975756091988L) - } - - def serializeToBase64[T](o: T): String = { - val baos = new ByteArrayOutputStream - val oos = new ObjectOutputStream(baos) - try { - oos.writeObject(o) - Base64.getEncoder.encodeToString(baos.toByteArray) - } finally { - oos.close() + // Ensure we are not serializable after the dereference (type "Object" is not serializable) + intercept[NotSerializableException] { + val oos2 = new ObjectOutputStream(new ByteArrayOutputStream()) + oos2.writeObject(lazyval) } } - - def deserializeFromBase64[T](base64Str: String): T = { - val bytes = Base64.getDecoder.decode(base64Str) - val bais = new ByteArrayInputStream(bytes) - val ois = new ObjectInputStream(bais) - ois.readObject().asInstanceOf[T] - } }