From c0646b61f1612ebd9ca1d270cbb33f0a680cc1f7 Mon Sep 17 00:00:00 2001 From: Martin Mauch Date: Tue, 19 May 2020 12:00:12 +0200 Subject: [PATCH] Update Scalatest to 3.1.1 and run Scalafix --- build.sbt | 3 ++- .../spark/testing/ArtisinalStreamingTest.scala | 4 ++-- .../holdenkarau/spark/testing/HDFSClusterTest.scala | 4 ++-- .../spark/testing/MultipleDataFrameSuites.scala | 4 ++-- .../holdenkarau/spark/testing/PerTestSampleTest.scala | 4 ++-- .../com/holdenkarau/spark/testing/PerfSampleTest.scala | 4 ++-- .../spark/testing/SampleDataFrameTest.scala | 4 ++-- .../com/holdenkarau/spark/testing/SampleRDDTest.scala | 4 ++-- .../spark/testing/SampleScalaCheckTest.scala | 10 +++++----- .../spark/testing/SampleStreamingActionTest.scala | 4 ++-- .../spark/testing/SampleStreamingTest.scala | 4 ++-- .../com/holdenkarau/spark/testing/UtilsTest.scala | 4 ++-- .../com/holdenkarau/spark/testing/PrettifyTest.scala | 6 +++--- .../spark/testing/SampleDatasetGeneratorTest.scala | 4 ++-- .../holdenkarau/spark/testing/SampleDatasetTest.scala | 4 ++-- .../spark/testing/SampleMiniClusterTest.scala | 4 ++-- .../holdenkarau/spark/testing/YARNClusterTest.scala | 5 +++-- .../spark/testing/DatasetGeneratorSizeSpecial.scala | 4 ++-- .../holdenkarau/spark/testing/MLScalaCheckTest.scala | 4 ++-- .../spark/testing/PerTestSparkContextTest.scala | 4 ++-- .../spark/testing/StructuredStreamingSampleTests.scala | 4 ++-- 21 files changed, 47 insertions(+), 45 deletions(-) diff --git a/build.sbt b/build.sbt index 945c38d7..30611577 100644 --- a/build.sbt +++ b/build.sbt @@ -226,7 +226,8 @@ val coreTestSources = unmanagedSourceDirectories in Test := { // additional libraries lazy val commonDependencies = Seq( - "org.scalatest" %% "scalatest" % "3.0.8", + "org.scalatest" %% "scalatest" % "3.1.1", + "org.scalatestplus" %% "scalatestplus-scalacheck" % "3.1.0.0-RC2", "io.github.nicolasstucki" %% "multisets" % "0.4", "org.scalacheck" %% "scalacheck" % "1.14.0", "junit" % "junit" % "4.12", diff --git a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/ArtisinalStreamingTest.scala b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/ArtisinalStreamingTest.scala index 8e302529..ea368fe0 100644 --- a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/ArtisinalStreamingTest.scala +++ b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/ArtisinalStreamingTest.scala @@ -25,8 +25,8 @@ import org.apache.spark._ import org.apache.spark.rdd.RDD import org.apache.spark.SparkContext._ -import org.scalatest.FunSuite import org.scalatest.exceptions.TestFailedException +import org.scalatest.funsuite.AnyFunSuite /** * ArtisinalStreamingTest illustrates how to write a streaming test @@ -36,7 +36,7 @@ import org.scalatest.exceptions.TestFailedException * This does not use a manual clock and instead uses the kind of sketchy * sleep approach. Instead please look at [[SampleStreamingTest]]. */ -class ArtisinalStreamingTest extends FunSuite with SharedSparkContext { +class ArtisinalStreamingTest extends AnyFunSuite with SharedSparkContext { // tag::createQueueStream[] def makeSimpleQueueStream(ssc: StreamingContext) = { val input = List(List("hi"), List("happy pandas", "sad pandas")) diff --git a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/HDFSClusterTest.scala b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/HDFSClusterTest.scala index 3555ae82..217aaaa1 100644 --- a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/HDFSClusterTest.scala +++ b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/HDFSClusterTest.scala @@ -7,9 +7,9 @@ import com.holdenkarau.spark.testing.{RDDComparisons, SharedSparkContext} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.rdd.RDD -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class HDFSClusterTest extends FunSuite with SharedSparkContext with RDDComparisons { +class HDFSClusterTest extends AnyFunSuite with SharedSparkContext with RDDComparisons { var hdfsCluster: HDFSCluster = null diff --git a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/MultipleDataFrameSuites.scala b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/MultipleDataFrameSuites.scala index fd9a7a41..6352d67c 100644 --- a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/MultipleDataFrameSuites.scala +++ b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/MultipleDataFrameSuites.scala @@ -1,8 +1,8 @@ package com.holdenkarau.spark.testing -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class MultipleDataFrameSuites extends FunSuite with DataFrameSuiteBase { +class MultipleDataFrameSuites extends AnyFunSuite with DataFrameSuiteBase { test("test nothing") { assert(1 === 1) } diff --git a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/PerTestSampleTest.scala b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/PerTestSampleTest.scala index 44328056..0ef017a3 100644 --- a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/PerTestSampleTest.scala +++ b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/PerTestSampleTest.scala @@ -16,14 +16,14 @@ */ package com.holdenkarau.spark.testing -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite /** * Illustrate using per-test sample test. This is the one to use * when your tests may be destructive to the Spark context * (e.g. stopping it) */ -class PerTestSampleTest extends FunSuite with PerTestSparkContext { +class PerTestSampleTest extends AnyFunSuite with PerTestSparkContext { test("sample test stops a context") { sc.stop() diff --git a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/PerfSampleTest.scala b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/PerfSampleTest.scala index 5ef11f8f..75e867c9 100644 --- a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/PerfSampleTest.scala +++ b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/PerfSampleTest.scala @@ -19,14 +19,14 @@ package com.holdenkarau.spark.testing import java.nio.file.Files import org.apache.spark._ -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite /** * Illustrate using per-test sample test. This is the one to use * when your tests may be destructive to the Spark context * (e.g. stopping it) */ -class PerfSampleTest extends FunSuite with PerTestSparkContext { +class PerfSampleTest extends AnyFunSuite with PerTestSparkContext { val tempPath = Files.createTempDirectory(null).toString() //tag::samplePerfTest[] diff --git a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleDataFrameTest.scala b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleDataFrameTest.scala index 19ce219b..8907cd03 100644 --- a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleDataFrameTest.scala +++ b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleDataFrameTest.scala @@ -20,9 +20,9 @@ import java.sql.Timestamp import org.apache.spark.sql.Row import org.apache.spark.sql.types._ -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class SampleDataFrameTest extends FunSuite with DataFrameSuiteBase { +class SampleDataFrameTest extends AnyFunSuite with DataFrameSuiteBase { val byteArray = new Array[Byte](1) val diffByteArray = Array[Byte](192.toByte) val inputList = List( diff --git a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleRDDTest.scala b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleRDDTest.scala index a30e0719..36158466 100644 --- a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleRDDTest.scala +++ b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleRDDTest.scala @@ -19,9 +19,9 @@ package com.holdenkarau.spark.testing import scala.util.Random import org.apache.spark.rdd.RDD -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class SampleRDDTest extends FunSuite with SharedSparkContext with RDDComparisons { +class SampleRDDTest extends AnyFunSuite with SharedSparkContext with RDDComparisons { test("really simple transformation") { val input = List("hi", "hi holden", "bye") val expected = List(List("hi"), List("hi", "holden"), List("bye")) diff --git a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleScalaCheckTest.scala b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleScalaCheckTest.scala index 7391066b..d5cb30fa 100644 --- a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleScalaCheckTest.scala +++ b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleScalaCheckTest.scala @@ -21,10 +21,10 @@ import org.apache.spark.sql.{DataFrame, Row, SQLContext} import org.apache.spark.sql.types._ import org.scalacheck.{Arbitrary, Gen} import org.scalacheck.Prop.forAll -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.Checkers +import org.scalatest.funsuite.AnyFunSuite -class SampleScalaCheckTest extends FunSuite +class SampleScalaCheckTest extends AnyFunSuite with SharedSparkContext with RDDComparisons with Checkers { // tag::propertySample[] // A trivial property that the map doesn't change the number of elements @@ -257,7 +257,7 @@ class SampleScalaCheckTest extends FunSuite test("generate rdd of specific size") { implicit val generatorDrivenConfig = - PropertyCheckConfig(minSize = 10, maxSize = 20) + PropertyCheckConfiguration(minSize = 10, sizeRange = 10) val prop = forAll(RDDGenerator.genRDD[String](sc)(Arbitrary.arbitrary[String])){ rdd => rdd.count() <= 20 } @@ -333,7 +333,7 @@ class SampleScalaCheckTest extends FunSuite StructType(StructField("timestampType", TimestampType) :: Nil)) :: Nil test("second dataframe's evaluation has the same values as first") { implicit val generatorDrivenConfig = - PropertyCheckConfig(minSize = 1, maxSize = 1) + PropertyCheckConfiguration(minSize = 1, sizeRange = 0) val sqlContext = new SQLContext(sc) val dataframeGen = @@ -354,7 +354,7 @@ class SampleScalaCheckTest extends FunSuite } test("nullable fields contain null values as well") { implicit val generatorDrivenConfig = - PropertyCheckConfig(minSize = 1, maxSize = 1) + PropertyCheckConfiguration(minSize = 1, sizeRange = 0) val nullableFields = fields.map(f => f.copy(nullable = true, name = s"${f.name}Nullable")) val sqlContext = new SQLContext(sc) val allFields = fields ::: nullableFields diff --git a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleStreamingActionTest.scala b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleStreamingActionTest.scala index 9e1a99f7..26d6b15e 100644 --- a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleStreamingActionTest.scala +++ b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleStreamingActionTest.scala @@ -19,9 +19,9 @@ package com.holdenkarau.spark.testing import org.apache.spark._ import org.apache.spark.rdd.RDD import org.apache.spark.streaming.dstream._ -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class SampleStreamingActionTest extends FunSuite with StreamingActionBase { +class SampleStreamingActionTest extends AnyFunSuite with StreamingActionBase { test("a simple action") { val input = List(List("hi"), List("bye")) diff --git a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleStreamingTest.scala b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleStreamingTest.scala index af065ed7..0fdeb384 100644 --- a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleStreamingTest.scala +++ b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/SampleStreamingTest.scala @@ -21,9 +21,9 @@ import org.apache.spark.streaming.Seconds import org.apache.spark.streaming.dstream._ import org.scalactic.Equality import org.scalatest.exceptions.TestFailedException -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class SampleStreamingTest extends FunSuite with StreamingSuiteBase { +class SampleStreamingTest extends AnyFunSuite with StreamingSuiteBase { //tag::simpleStreamingTest[] test("really simple transformation") { diff --git a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/UtilsTest.scala b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/UtilsTest.scala index daa7f683..d4e7f85e 100644 --- a/core/src/test/1.3/scala/com/holdenkarau/spark/testing/UtilsTest.scala +++ b/core/src/test/1.3/scala/com/holdenkarau/spark/testing/UtilsTest.scala @@ -19,9 +19,9 @@ package com.holdenkarau.spark.testing import java.io._ import java.nio.file.Files -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class UtilsTest extends FunSuite { +class UtilsTest extends AnyFunSuite { test("test utils cleanup") { val tempDir = Utils.createTempDir() val tempPath = tempDir.toPath().toAbsolutePath().toString() diff --git a/core/src/test/1.6/scala/com/holdenkarau/spark/testing/PrettifyTest.scala b/core/src/test/1.6/scala/com/holdenkarau/spark/testing/PrettifyTest.scala index 3c419960..ad04600d 100644 --- a/core/src/test/1.6/scala/com/holdenkarau/spark/testing/PrettifyTest.scala +++ b/core/src/test/1.6/scala/com/holdenkarau/spark/testing/PrettifyTest.scala @@ -5,12 +5,12 @@ import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructT import org.scalacheck.Gen import org.scalacheck.Prop._ import org.scalacheck.util.Pretty -import org.scalatest.FunSuite import org.scalatest.exceptions.GeneratorDrivenPropertyCheckFailedException import org.scalatestplus.scalacheck.Checkers +import org.scalatest.funsuite.AnyFunSuite -class PrettifyTest extends FunSuite with SharedSparkContext with Checkers with Prettify { - implicit val propertyCheckConfig = PropertyCheckConfig(minSize = 2, maxSize = 2) +class PrettifyTest extends AnyFunSuite with SharedSparkContext with Checkers with Prettify { + implicit val propertyCheckConfig = PropertyCheckConfiguration(minSize = 2, sizeRange = 0) test("pretty output of DataFrame's check") { val schema = StructType(List(StructField("name", StringType), StructField("age", IntegerType))) diff --git a/core/src/test/1.6/scala/com/holdenkarau/spark/testing/SampleDatasetGeneratorTest.scala b/core/src/test/1.6/scala/com/holdenkarau/spark/testing/SampleDatasetGeneratorTest.scala index bfb65c36..c1fb1c88 100644 --- a/core/src/test/1.6/scala/com/holdenkarau/spark/testing/SampleDatasetGeneratorTest.scala +++ b/core/src/test/1.6/scala/com/holdenkarau/spark/testing/SampleDatasetGeneratorTest.scala @@ -3,10 +3,10 @@ package com.holdenkarau.spark.testing import org.apache.spark.sql.{Dataset, SQLContext} import org.scalacheck.{Gen, Arbitrary} import org.scalacheck.Prop.forAll -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.Checkers +import org.scalatest.funsuite.AnyFunSuite -class SampleDatasetGeneratorTest extends FunSuite +class SampleDatasetGeneratorTest extends AnyFunSuite with SharedSparkContext with Checkers { test("test generating Datasets[String]") { diff --git a/core/src/test/1.6/scala/com/holdenkarau/spark/testing/SampleDatasetTest.scala b/core/src/test/1.6/scala/com/holdenkarau/spark/testing/SampleDatasetTest.scala index 55ea5573..c7cab011 100644 --- a/core/src/test/1.6/scala/com/holdenkarau/spark/testing/SampleDatasetTest.scala +++ b/core/src/test/1.6/scala/com/holdenkarau/spark/testing/SampleDatasetTest.scala @@ -2,9 +2,9 @@ package com.holdenkarau.spark.testing import java.sql.Timestamp -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class SampleDatasetTest extends FunSuite with DatasetSuiteBase { +class SampleDatasetTest extends AnyFunSuite with DatasetSuiteBase { test("equal empty dataset") { import sqlContext.implicits._ diff --git a/core/src/test/1.6/scala/com/holdenkarau/spark/testing/SampleMiniClusterTest.scala b/core/src/test/1.6/scala/com/holdenkarau/spark/testing/SampleMiniClusterTest.scala index ee239cad..cd674a4f 100644 --- a/core/src/test/1.6/scala/com/holdenkarau/spark/testing/SampleMiniClusterTest.scala +++ b/core/src/test/1.6/scala/com/holdenkarau/spark/testing/SampleMiniClusterTest.scala @@ -17,10 +17,10 @@ package com.holdenkarau.spark.testing import org.apache.spark.rdd.RDD -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class SampleMiniClusterTest extends FunSuite with SharedMiniCluster { +class SampleMiniClusterTest extends AnyFunSuite with SharedMiniCluster { test("really simple transformation") { val input = List("hi", "hi holden", "bye") diff --git a/core/src/test/1.6/scala/com/holdenkarau/spark/testing/YARNClusterTest.scala b/core/src/test/1.6/scala/com/holdenkarau/spark/testing/YARNClusterTest.scala index 33156887..bb0ce4f1 100644 --- a/core/src/test/1.6/scala/com/holdenkarau/spark/testing/YARNClusterTest.scala +++ b/core/src/test/1.6/scala/com/holdenkarau/spark/testing/YARNClusterTest.scala @@ -1,10 +1,11 @@ package com.holdenkarau.spark.testing import org.apache.spark.{SparkConf, SparkContext} -import org.scalatest.{BeforeAndAfterAll, FunSuite} +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite -class YARNClusterTest extends FunSuite with BeforeAndAfterAll { +class YARNClusterTest extends AnyFunSuite with BeforeAndAfterAll { var yarnCluster: YARNCluster = null var sc: SparkContext = null diff --git a/core/src/test/2.0/scala/com/holdenkarau/spark/testing/DatasetGeneratorSizeSpecial.scala b/core/src/test/2.0/scala/com/holdenkarau/spark/testing/DatasetGeneratorSizeSpecial.scala index 7bac2d8d..331247ce 100644 --- a/core/src/test/2.0/scala/com/holdenkarau/spark/testing/DatasetGeneratorSizeSpecial.scala +++ b/core/src/test/2.0/scala/com/holdenkarau/spark/testing/DatasetGeneratorSizeSpecial.scala @@ -3,10 +3,10 @@ package com.holdenkarau.spark.testing import org.apache.spark.sql.{Dataset, SQLContext} import org.scalacheck.{Gen, Arbitrary} import org.scalacheck.Prop.forAll -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.Checkers +import org.scalatest.funsuite.AnyFunSuite -class DatasetGeneratorSizeSpecial extends FunSuite +class DatasetGeneratorSizeSpecial extends AnyFunSuite with SharedSparkContext with Checkers { test("test generating sized Datasets[Custom Class]") { diff --git a/core/src/test/2.0/scala/com/holdenkarau/spark/testing/MLScalaCheckTest.scala b/core/src/test/2.0/scala/com/holdenkarau/spark/testing/MLScalaCheckTest.scala index abaea76e..9a6ba910 100644 --- a/core/src/test/2.0/scala/com/holdenkarau/spark/testing/MLScalaCheckTest.scala +++ b/core/src/test/2.0/scala/com/holdenkarau/spark/testing/MLScalaCheckTest.scala @@ -4,10 +4,10 @@ import org.apache.spark.ml.linalg.SQLDataTypes.{MatrixType, VectorType} import org.apache.spark.sql.SQLContext import org.apache.spark.sql.types.{StructField, StructType} import org.scalacheck.Prop.forAll -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.Checkers +import org.scalatest.funsuite.AnyFunSuite -class MLScalaCheckTest extends FunSuite with SharedSparkContext with Checkers { +class MLScalaCheckTest extends AnyFunSuite with SharedSparkContext with Checkers { // re-use the spark context override implicit def reuseContextIfPossible: Boolean = false diff --git a/core/src/test/2.0/scala/com/holdenkarau/spark/testing/PerTestSparkContextTest.scala b/core/src/test/2.0/scala/com/holdenkarau/spark/testing/PerTestSparkContextTest.scala index 54b36a69..59ed2cc3 100644 --- a/core/src/test/2.0/scala/com/holdenkarau/spark/testing/PerTestSparkContextTest.scala +++ b/core/src/test/2.0/scala/com/holdenkarau/spark/testing/PerTestSparkContextTest.scala @@ -19,14 +19,14 @@ package com.holdenkarau.spark.testing import java.nio.file.Files import org.apache.spark._ -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite /** * Illustrate using per-test sample test. This is the one to use * when your tests may be destructive to the Spark context * (e.g. stopping it) */ -class PerTestSparkContextTest extends FunSuite with PerTestSparkContext { +class PerTestSparkContextTest extends AnyFunSuite with PerTestSparkContext { val tempPath = Files.createTempDirectory(null).toString() //tag::samplePerfTest[] diff --git a/core/src/test/2.2/scala/com/holdenkarau/spark/testing/StructuredStreamingSampleTests.scala b/core/src/test/2.2/scala/com/holdenkarau/spark/testing/StructuredStreamingSampleTests.scala index 49138ea2..99ad3cad 100644 --- a/core/src/test/2.2/scala/com/holdenkarau/spark/testing/StructuredStreamingSampleTests.scala +++ b/core/src/test/2.2/scala/com/holdenkarau/spark/testing/StructuredStreamingSampleTests.scala @@ -2,10 +2,10 @@ package com.holdenkarau.spark.testing import org.apache.spark.sql._ -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite class StructuredStreamingTests - extends FunSuite with SharedSparkContext with StructuredStreamingBase { + extends AnyFunSuite with SharedSparkContext with StructuredStreamingBase { // re-use the spark context override implicit def reuseContextIfPossible: Boolean = true