-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* add the JavaImplicitConverters trait * add the SparkSpec trait * add the SparkStreamingSpec trait * add the FileSystemSpec trait * update the documentation
- Loading branch information
Showing
14 changed files
with
526 additions
and
72 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,6 +2,6 @@ | |
| title: Advanced | ||
| layout: default | ||
| has_children: true | ||
| nav_order: 6 | ||
| nav_order: 7 | ||
| --- | ||
| # Advanced | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,7 +2,7 @@ | |
| title: Configuration | ||
| layout: default | ||
| has_children: true | ||
| nav_order: 5 | ||
| nav_order: 6 | ||
| --- | ||
| # Configuration | ||
|
|
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| --- | ||
| title: Writing tests | ||
| layout: default | ||
| nav_order: 5 | ||
| --- | ||
| # Writing tests | ||
| <details open markdown="block"> | ||
| <summary> | ||
| Table of contents | ||
| </summary> | ||
| {: .text-delta } | ||
| 1. TOC | ||
| {:toc} | ||
| </details> | ||
|
|
||
| --- | ||
|
|
||
| Data I/O offers a separate library with utility traits and methods designed to facilitate testing Scala/Spark SQL applications. | ||
|
|
||
| ## Installation | ||
|
|
||
| Published releases are available on GitHub Packages, in the AmadeusITGroup repository. | ||
|
|
||
| Using Maven: | ||
|
|
||
| ```xml | ||
| <dependency> | ||
| <groupId>com.amadeus.dataio</groupId> | ||
| <artifactId>dataio-test</artifactId> | ||
| <version>x.x.x</version> | ||
| </dependency> | ||
| ``` | ||
|
|
||
| ## Overview | ||
|
|
||
|
|
||
| ### Interacting with the file system | ||
| The `FileSystemSpec` trait provides the Hadoop `LocalFileSystem` for tests needing direct access to an instance of `FileSystem`. | ||
|
|
||
| Example: | ||
|
|
||
| ```scala | ||
|
|
||
| import com.amadeus.dataio.test._ | ||
| import org.scalatest.flatspec.AnyFlatSpec | ||
|
|
||
| case class MyAppTest extends AnyFlatSpec with FileSystemSpec { | ||
| "MyAppTest" should "do something" in { | ||
| assert(fs.exists("file:///my_file.txt")) | ||
| } | ||
| } | ||
| ``` | ||
|
|
||
|
|
||
| ### Interacting with a SparkSession | ||
| The `SparkSpec` trait provides a local Spark session and helper functions for Spark tests: | ||
| - `getTestName: String`: Returns the test suite's name. | ||
| - `collectData(path: String, format: String, schema: Option[String] = None): Array[String])`: Collects data from the file system. | ||
|
|
||
| Note that extending this trait, you will have to override the getTestName: String function. | ||
|
|
||
| Example: | ||
|
|
||
| ```scala | ||
|
|
||
| import com.amadeus.dataio.test._ | ||
| import org.scalatest.flatspec.AnyFlatSpec | ||
|
|
||
| case class MyAppTest extends AnyFlatSpec with SparkSpec { | ||
| override def getTestName = "MyAppTest" | ||
|
|
||
| "MyAppTest" should "do something" in { | ||
| spark.read.format("csv").load("my_data.csv") | ||
| collectData | ||
| } | ||
| } | ||
| ``` | ||
|
|
||
|
|
||
| ### Interacting with a Streaming context | ||
| The `SparkStreamingSpec` trait provides a local Spark session and helper functions for Spark Streaming tests: | ||
| - `enableSparkStreamingSchemaInference(): Unit`: Enables Spark streaming schema inference. | ||
| - `collectDataStream(dataFrame: DataFrame): Array[String]`: Collects data from a DataFrame read from a stream using an in-memory sink. | ||
|
|
||
|
|
||
| ### Implicitly converting Scala Maps and Lists in Java equivalents | ||
| It it sometimes necessary to build complex map structures while building `Typesafe Config` objects, requiring redundant Scala-to-Java conversions. | ||
|
|
||
| To simplify this, you may extend the `JavaImplicitConverters` trait. | ||
|
|
||
| Example: | ||
|
|
||
| ```scala | ||
|
|
||
| import com.amadeus.dataio.test._ | ||
| import com.typesafe.config.ConfigFactory | ||
| import org.scalatest.flatspec.AnyFlatSpec | ||
|
|
||
| case class MyAppTest extends AnyFlatSpec with JavaImplicitConverters { | ||
| "MyAppTest" should "do something" in { | ||
| ConfigFactory.parseMap( | ||
| Map("NodeName" -> Seq(Map("Type" -> "com.Entity"), Map("Type" -> "com.Entity"))) | ||
| ) | ||
| } | ||
| } | ||
| ``` | ||
|
|
36 changes: 36 additions & 0 deletions
36
test/src/main/scala/com/amadeus/dataio/test/FileSystemSpec.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| package com.amadeus.dataio.test | ||
|
|
||
| import org.apache.hadoop.conf.Configuration | ||
| import org.apache.hadoop.fs.{FileSystem, Path} | ||
| import org.scalatest.{BeforeAndAfter, TestSuite} | ||
|
|
||
| /** | ||
| * Provides the Hadoop LocalFileSystem for tests needing direct access to an instance of FileSystem. | ||
| * | ||
| * Provides a dedicated instance initialized before each test and automatically closed after each test, providing as | ||
| * much isolation as possible between tests. It also deletes the dataio-test temporary directory (/tmp/dataio-test/) and | ||
| * sub-directories, before closing the FileSystem. | ||
| * | ||
| * e.g. | ||
| * {{{ | ||
| * class MyClassTest extends WordSpec with FileSystemSpec{ | ||
| * // provided by FileSystemSpec: | ||
| * // fs: FileSystem | ||
| * // val tmpPath: String = "file:///tmp/dataio-test/" | ||
| * } | ||
| * }}} | ||
| */ | ||
| trait FileSystemSpec extends TestSuite with BeforeAndAfter { | ||
| val tmpPath = "file:///tmp/dataio-test/" | ||
|
|
||
| var fs: FileSystem = _ | ||
|
|
||
| before { | ||
| fs = FileSystem.newInstance(new Configuration()) | ||
| } | ||
|
|
||
| after { | ||
| fs.delete(new Path(tmpPath), true) | ||
| fs.close() | ||
| } | ||
| } |
Oops, something went wrong.