From 283ee2cbe2845f78ed31f72c5a7c3b7045b81b16 Mon Sep 17 00:00:00 2001 From: Leonard Wolters Date: Thu, 2 Nov 2023 10:27:11 +0100 Subject: [PATCH 01/12] Fixing dependency of Pekko --- build.sbt | 2 +- project/Build.scala | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 13ae4f66..debfe7e5 100644 --- a/build.sbt +++ b/build.sbt @@ -75,7 +75,7 @@ lazy val client: Project = (project in file("client")) "org.apache.pekko" %% "pekko-http" % PekkoHttpVersion, "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5", "joda-time" % "joda-time" % "2.12.2" - ) ++ testDependencies.map(_ % Test) + ) ++ Seq("org.apache.pekko" %% "pekko-testkit" % PekkoVersion % Test) ++ Build.testDependencies.map(_ % Test) ) lazy val dsl = (project in file("dsl")) diff --git a/project/Build.scala b/project/Build.scala index 60d279aa..f23b7744 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -5,7 +5,5 @@ object Build { val PekkoVersion = "1.0.1" val PekkoHttpVersion = "1.0.0" - val testDependencies = Seq("org.scalatest" %% "scalatest" % "3.2.15", - "org.apache.pekko" %% "pekko-testkit" % PekkoVersion, - "ch.qos.logback" % "logback-classic" % "1.4.6") + val testDependencies = Seq("org.scalatest" %% "scalatest" % "3.2.15", "ch.qos.logback" % "logback-classic" % "1.4.7") } From 85f200bfc47e498a7510cec847d9b30a3b87612d Mon Sep 17 00:00:00 2001 From: Leonard Wolters Date: Thu, 2 Nov 2023 10:27:49 +0100 Subject: [PATCH 02/12] Cleaning --- build.sbt | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/build.sbt b/build.sbt index debfe7e5..975f4c22 100644 --- a/build.sbt +++ b/build.sbt @@ -1,4 +1,4 @@ -import Build._ +import Build.* // Scala Formatting ThisBuild / scalafmtVersion := "1.5.1" @@ -19,15 +19,7 @@ lazy val root = (project in file(".")) scalaVersion := "2.13.8", crossScalaVersions := List("2.12.13", "2.13.8"), javacOptions ++= Seq("-g", "-Xlint:unchecked", "-Xlint:deprecation", "-source", "11", "-target", "11"), - scalacOptions ++= Seq( -// "-Wconf:cat=deprecation:ws,any:e", -// "-target:jvm-11", - "-unchecked", - "-deprecation", - "-feature", - "-language:_", - "-encoding", - "UTF-8"), + scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-language:_", "-encoding", "UTF-8"), publishTo := { val nexus = "https://oss.sonatype.org/" if (version.value.trim.endsWith("SNAPSHOT")) @@ -79,6 +71,7 @@ lazy val client: Project = (project in file("client")) ) lazy val dsl = (project in file("dsl")) + .dependsOn(client, client % "test->test", testkit % Test) .configs(Config.CustomIntegrationTest) .settings(Config.testSettings: _*) .settings( @@ -86,12 +79,11 @@ lazy val dsl = (project in file("dsl")) sbtrelease.ReleasePlugin.autoImport.releasePublishArtifactsAction := PgpKeys.publishSigned.value, libraryDependencies ++= Seq("com.google.guava" % "guava" % "23.0") ) - .dependsOn(client, client % "test->test", testkit % Test) lazy val testkit = (project in file("testkit")) + .dependsOn(client) .settings( name := "testkit", sbtrelease.ReleasePlugin.autoImport.releasePublishArtifactsAction := PgpKeys.publishSigned.value, libraryDependencies ++= Build.testDependencies ) - .dependsOn(client) From b3e25e502a2c548a56422bc1a5d90d11b2873898 Mon Sep 17 00:00:00 2001 From: Leonard Wolters Date: Thu, 2 Nov 2023 10:33:29 +0100 Subject: [PATCH 03/12] Cleaning up code (removing support for scala 2.12) --- .../clickhouse/balancing/HostBalancer.scala | 7 +++---- .../clickhouse/internal/QuerySettings.scala | 7 +++---- .../ClickhouseIndexingSubscriberTest.scala | 18 ++++++++++-------- .../execution/ClickhouseQueryExecutor.scala | 10 +++++++--- .../dsl/execution/QueryExecutor.scala | 7 +++++-- .../language/ClickhouseTokenizerModule.scala | 3 +-- .../dsl/misc/QueryImprovements.scala | 18 +++++++++--------- .../com/crobox/clickhouse/dsl/package.scala | 6 +++--- .../crobox/clickhouse/DslIntegrationSpec.scala | 4 ++-- 9 files changed, 43 insertions(+), 37 deletions(-) diff --git a/client/src/main/scala/com/crobox/clickhouse/balancing/HostBalancer.scala b/client/src/main/scala/com/crobox/clickhouse/balancing/HostBalancer.scala index 52d151c3..bc57d921 100644 --- a/client/src/main/scala/com/crobox/clickhouse/balancing/HostBalancer.scala +++ b/client/src/main/scala/com/crobox/clickhouse/balancing/HostBalancer.scala @@ -1,18 +1,17 @@ package com.crobox.clickhouse.balancing -import org.apache.pekko.actor.ActorSystem -import org.apache.pekko.http.scaladsl.model._ import com.crobox.clickhouse.balancing.Connection.{BalancingHosts, ClusterAware, ConnectionType, SingleHost} import com.crobox.clickhouse.balancing.discovery.ConnectionManagerActor import com.crobox.clickhouse.balancing.discovery.health.ClickhouseHostHealth import com.crobox.clickhouse.internal.ClickhouseHostBuilder import com.typesafe.config.Config import com.typesafe.scalalogging.LazyLogging +import org.apache.pekko.actor.ActorSystem +import org.apache.pekko.http.scaladsl.model._ -import scala.collection.JavaConverters._ import scala.concurrent.duration._ import scala.concurrent.{ExecutionContext, Future} -//import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters._ trait HostBalancer extends LazyLogging { def nextHost: Future[Uri] diff --git a/client/src/main/scala/com/crobox/clickhouse/internal/QuerySettings.scala b/client/src/main/scala/com/crobox/clickhouse/internal/QuerySettings.scala index c07d0275..6b09f78b 100644 --- a/client/src/main/scala/com/crobox/clickhouse/internal/QuerySettings.scala +++ b/client/src/main/scala/com/crobox/clickhouse/internal/QuerySettings.scala @@ -1,12 +1,11 @@ package com.crobox.clickhouse.internal -import org.apache.pekko.http.scaladsl.model.Uri.Query -import org.apache.pekko.http.scaladsl.model.headers.HttpEncoding import com.crobox.clickhouse.internal.QuerySettings._ import com.typesafe.config.Config +import org.apache.pekko.http.scaladsl.model.Uri.Query +import org.apache.pekko.http.scaladsl.model.headers.HttpEncoding -//import scala.jdk.CollectionConverters._ -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import scala.util.Try case class QuerySettings(readOnly: ReadOnlySetting = AllQueries, diff --git a/client/src/test/scala/com/crobox/clickhouse/stream/ClickhouseIndexingSubscriberTest.scala b/client/src/test/scala/com/crobox/clickhouse/stream/ClickhouseIndexingSubscriberTest.scala index 42d0c50e..36508209 100644 --- a/client/src/test/scala/com/crobox/clickhouse/stream/ClickhouseIndexingSubscriberTest.scala +++ b/client/src/test/scala/com/crobox/clickhouse/stream/ClickhouseIndexingSubscriberTest.scala @@ -1,8 +1,8 @@ package com.crobox.clickhouse.stream -import org.apache.pekko.stream.scaladsl._ import com.crobox.clickhouse.internal.QuerySettings import com.crobox.clickhouse.{ClickhouseClient, ClickhouseClientAsyncSpec} +import org.apache.pekko.stream.scaladsl._ import org.scalatest.concurrent.{Eventually, ScalaFutures} import scala.concurrent.duration._ @@ -54,19 +54,21 @@ class ClickhouseIndexingSubscriberTest extends ClickhouseClientAsyncSpec with Sc ) ) - def parsedInserts(key: String) = unparsedInserts(key).map( - _.mapValues({ // do NOT change to .view.mapValues given compilation errors for scala 2.12.+ - case value: Int => value.toString - case value: String => "\"" + value + "\"" - case value: IndexedSeq[_] => "[" + value.mkString(", ") + "]" - }).map { case (k, v) => s""""$k" : $v""" } + def parsedInserts(key: String): Seq[String] = unparsedInserts(key).map( + _.view + .mapValues({ + case value: Int => value.toString + case value: String => "\"" + value + "\"" + case value: IndexedSeq[_] => "[" + value.mkString(", ") + "]" + }) + .map { case (k, v) => s""""$k" : $v""" } .mkString(", ") ) it should "index items" in { val inserts = parsedInserts("two") val res = Source - .fromIterator(() => inserts.toIterator) // do NOT change to .iterator given compilation errors for scala 2.12.+ + .fromIterator(() => inserts.iterator) .map(data => Insert("test.insert", "{" + data + "}")) .runWith(ClickhouseSink.toSink(config, client, Some("no-overrides"))) Await.ready(res, 5.seconds) diff --git a/dsl/src/main/scala/com/crobox/clickhouse/dsl/execution/ClickhouseQueryExecutor.scala b/dsl/src/main/scala/com/crobox/clickhouse/dsl/execution/ClickhouseQueryExecutor.scala index d86f36d1..8d375580 100644 --- a/dsl/src/main/scala/com/crobox/clickhouse/dsl/execution/ClickhouseQueryExecutor.scala +++ b/dsl/src/main/scala/com/crobox/clickhouse/dsl/execution/ClickhouseQueryExecutor.scala @@ -1,11 +1,12 @@ package com.crobox.clickhouse.dsl.execution -import org.apache.pekko.stream.scaladsl.Source +import com.crobox.clickhouse import com.crobox.clickhouse.ClickhouseClient import com.crobox.clickhouse.dsl.language.{ClickhouseTokenizerModule, TokenizeContext, TokenizerModule} import com.crobox.clickhouse.dsl.{Query, Table} import com.crobox.clickhouse.internal.QuerySettings import com.crobox.clickhouse.internal.progress.QueryProgress.QueryProgress +import org.apache.pekko.stream.scaladsl.Source import spray.json.{JsonReader, _} import scala.concurrent.{ExecutionContext, Future} @@ -14,8 +15,11 @@ trait ClickhouseQueryExecutor extends QueryExecutor { self: TokenizerModule => implicit val client: ClickhouseClient - def execute[V: JsonReader](query: Query)(implicit executionContext: ExecutionContext, - settings: QuerySettings = QuerySettings()): Future[QueryResult[V]] = { + override val serverVersion: clickhouse.ClickhouseServerVersion = client.serverVersion + + override def execute[V: JsonReader]( + query: Query + )(implicit executionContext: ExecutionContext, settings: QuerySettings = QuerySettings()): Future[QueryResult[V]] = { import QueryResult._ val queryResult = client.query(toSql(query.internalQuery)(ctx = TokenizeContext(client.serverVersion))) queryResult.map(_.parseJson.convertTo[QueryResult[V]]) diff --git a/dsl/src/main/scala/com/crobox/clickhouse/dsl/execution/QueryExecutor.scala b/dsl/src/main/scala/com/crobox/clickhouse/dsl/execution/QueryExecutor.scala index 3430d4a5..88166fea 100644 --- a/dsl/src/main/scala/com/crobox/clickhouse/dsl/execution/QueryExecutor.scala +++ b/dsl/src/main/scala/com/crobox/clickhouse/dsl/execution/QueryExecutor.scala @@ -1,5 +1,6 @@ package com.crobox.clickhouse.dsl.execution +import com.crobox.clickhouse.ClickhouseServerVersion import com.crobox.clickhouse.dsl.language.TokenizerModule import com.crobox.clickhouse.dsl.{Query, Table} import com.crobox.clickhouse.internal.QuerySettings @@ -12,9 +13,11 @@ import scala.concurrent.{ExecutionContext, Future} */ trait QueryExecutor { self: TokenizerModule => + val serverVersion: ClickhouseServerVersion + def execute[V: JsonReader](query: Query)(implicit executionContext: ExecutionContext, - settings: QuerySettings): Future[QueryResult[V]] + settings: QuerySettings = QuerySettings()): Future[QueryResult[V]] def insert[V: JsonWriter](table: Table, values: Seq[V])(implicit executionContext: ExecutionContext, - settings: QuerySettings): Future[String] + settings: QuerySettings = QuerySettings()): Future[String] } diff --git a/dsl/src/main/scala/com/crobox/clickhouse/dsl/language/ClickhouseTokenizerModule.scala b/dsl/src/main/scala/com/crobox/clickhouse/dsl/language/ClickhouseTokenizerModule.scala index 228b1456..3cab11b1 100644 --- a/dsl/src/main/scala/com/crobox/clickhouse/dsl/language/ClickhouseTokenizerModule.scala +++ b/dsl/src/main/scala/com/crobox/clickhouse/dsl/language/ClickhouseTokenizerModule.scala @@ -9,8 +9,7 @@ import com.typesafe.scalalogging.Logger import org.joda.time.{DateTime, DateTimeZone} import org.slf4j.LoggerFactory -import scala.collection.JavaConverters._ -//import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters._ case class TokenizeContext(version: ClickhouseServerVersion, var joinNr: Int = 0, diff --git a/dsl/src/main/scala/com/crobox/clickhouse/dsl/misc/QueryImprovements.scala b/dsl/src/main/scala/com/crobox/clickhouse/dsl/misc/QueryImprovements.scala index 0c7ccddb..77383c5d 100644 --- a/dsl/src/main/scala/com/crobox/clickhouse/dsl/misc/QueryImprovements.scala +++ b/dsl/src/main/scala/com/crobox/clickhouse/dsl/misc/QueryImprovements.scala @@ -1,7 +1,7 @@ package com.crobox.clickhouse.dsl.misc import com.crobox.clickhouse.dsl.Query -import com.crobox.clickhouse.dsl.execution.{ClickhouseQueryExecutor, QueryResult} +import com.crobox.clickhouse.dsl.execution.{QueryExecutor, QueryResult} import com.crobox.clickhouse.dsl.language.{ClickhouseTokenizerModule, TokenizeContext} import com.typesafe.scalalogging.LazyLogging import spray.json.JsonReader @@ -15,33 +15,33 @@ object QueryImprovements extends LazyLogging { def executeWithLogging[V: JsonReader](debug: Boolean)( implicit executionContext: ExecutionContext, - clickhouseExecutor: ClickhouseQueryExecutor + clickhouseExecutor: QueryExecutor ): Future[QueryResult[V]] = { if (debug) logger.info( - s"SQL: ${tokenizer.toSql(query.internalQuery)(TokenizeContext(clickhouseExecutor.client.serverVersion))}" + s"SQL: ${tokenizer.toSql(query.internalQuery)(TokenizeContext(clickhouseExecutor.serverVersion))}" ) clickhouseExecutor.execute(query) } def executeWithLogging[V: JsonReader](traceId: String)( implicit executionContext: ExecutionContext, - clickhouseExecutor: ClickhouseQueryExecutor + clickhouseExecutor: QueryExecutor ): Future[QueryResult[V]] = { logger.info( - s"[$traceId] ${tokenizer.toSql(query.internalQuery)(TokenizeContext(clickhouseExecutor.client.serverVersion))}" + s"[$traceId] ${tokenizer.toSql(query.internalQuery)(TokenizeContext(clickhouseExecutor.serverVersion))}" ) clickhouseExecutor.execute(query) } def executeWithLogging[V: JsonReader](traceId: Option[String])( implicit executionContext: ExecutionContext, - clickhouseExecutor: ClickhouseQueryExecutor + clickhouseExecutor: QueryExecutor ): Future[QueryResult[V]] = { traceId.foreach( id => logger.info( - s"[$id] ${tokenizer.toSql(query.internalQuery)(TokenizeContext(clickhouseExecutor.client.serverVersion))}" + s"[$id] ${tokenizer.toSql(query.internalQuery)(TokenizeContext(clickhouseExecutor.serverVersion))}" ) ) clickhouseExecutor.execute(query) @@ -49,10 +49,10 @@ object QueryImprovements extends LazyLogging { def executeWithLogging[V: JsonReader]( implicit executionContext: ExecutionContext, - clickhouseExecutor: ClickhouseQueryExecutor + clickhouseExecutor: QueryExecutor ): Future[QueryResult[V]] = { logger.info( - s"SQL: ${tokenizer.toSql(query.internalQuery)(TokenizeContext(clickhouseExecutor.client.serverVersion))}" + s"SQL: ${tokenizer.toSql(query.internalQuery)(TokenizeContext(clickhouseExecutor.serverVersion))}" ) clickhouseExecutor.execute(query) } diff --git a/dsl/src/main/scala/com/crobox/clickhouse/dsl/package.scala b/dsl/src/main/scala/com/crobox/clickhouse/dsl/package.scala index 39315bd7..48674e35 100644 --- a/dsl/src/main/scala/com/crobox/clickhouse/dsl/package.scala +++ b/dsl/src/main/scala/com/crobox/clickhouse/dsl/package.scala @@ -1,7 +1,7 @@ package com.crobox.clickhouse import com.crobox.clickhouse.dsl.column.ClickhouseColumnFunctions -import com.crobox.clickhouse.dsl.execution.{ClickhouseQueryExecutor, QueryResult} +import com.crobox.clickhouse.dsl.execution.{ClickhouseQueryExecutor, QueryExecutor, QueryResult} import com.crobox.clickhouse.dsl.marshalling.{QueryValue, QueryValueFormats} import spray.json.{JsonReader, JsonWriter} @@ -18,7 +18,7 @@ package object dsl extends ClickhouseColumnFunctions with QueryFactory with Quer def execute[V: JsonReader]( implicit executionContext: ExecutionContext, - clickhouseExecutor: ClickhouseQueryExecutor + clickhouseExecutor: QueryExecutor ): Future[QueryResult[V]] = clickhouseExecutor.execute(query) } @@ -26,7 +26,7 @@ package object dsl extends ClickhouseColumnFunctions with QueryFactory with Quer def into(table: Table)( implicit executionContext: ExecutionContext, - clickhouseExecutor: ClickhouseQueryExecutor + clickhouseExecutor: QueryExecutor ): Future[String] = clickhouseExecutor.insert(table, values) } diff --git a/dsl/src/test/scala/com/crobox/clickhouse/DslIntegrationSpec.scala b/dsl/src/test/scala/com/crobox/clickhouse/DslIntegrationSpec.scala index 61511b75..8f7bc072 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/DslIntegrationSpec.scala +++ b/dsl/src/test/scala/com/crobox/clickhouse/DslIntegrationSpec.scala @@ -1,7 +1,7 @@ package com.crobox.clickhouse import com.crobox.clickhouse.dsl._ -import com.crobox.clickhouse.dsl.execution.ClickhouseQueryExecutor +import com.crobox.clickhouse.dsl.execution.{ClickhouseQueryExecutor, QueryExecutor} import com.crobox.clickhouse.dsl.language.{ClickhouseTokenizerModule, TokenizeContext} import com.crobox.clickhouse.dsl.schemabuilder.{CreateTable, Engine} import com.crobox.clickhouse.testkit.ClickhouseSpec @@ -30,7 +30,7 @@ trait DslIntegrationSpec implicit val ec: ExecutionContext - implicit lazy val chExecutor: ClickhouseQueryExecutor = ClickhouseQueryExecutor.default(clickClient) + implicit lazy val chExecutor: QueryExecutor = ClickhouseQueryExecutor.default(clickClient) override def beforeAll(): Unit = { super.beforeAll() From 82de28ef0a864e220394eb177b1bd4662b42c8c4 Mon Sep 17 00:00:00 2001 From: Leonard Wolters Date: Thu, 2 Nov 2023 10:35:26 +0100 Subject: [PATCH 04/12] Removed support for executeWithProgress (removed dependency to pekko) --- .../execution/ClickhouseQueryExecutor.scala | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/dsl/src/main/scala/com/crobox/clickhouse/dsl/execution/ClickhouseQueryExecutor.scala b/dsl/src/main/scala/com/crobox/clickhouse/dsl/execution/ClickhouseQueryExecutor.scala index 8d375580..6c5650ad 100644 --- a/dsl/src/main/scala/com/crobox/clickhouse/dsl/execution/ClickhouseQueryExecutor.scala +++ b/dsl/src/main/scala/com/crobox/clickhouse/dsl/execution/ClickhouseQueryExecutor.scala @@ -5,8 +5,6 @@ import com.crobox.clickhouse.ClickhouseClient import com.crobox.clickhouse.dsl.language.{ClickhouseTokenizerModule, TokenizeContext, TokenizerModule} import com.crobox.clickhouse.dsl.{Query, Table} import com.crobox.clickhouse.internal.QuerySettings -import com.crobox.clickhouse.internal.progress.QueryProgress.QueryProgress -import org.apache.pekko.stream.scaladsl.Source import spray.json.{JsonReader, _} import scala.concurrent.{ExecutionContext, Future} @@ -25,15 +23,15 @@ trait ClickhouseQueryExecutor extends QueryExecutor { queryResult.map(_.parseJson.convertTo[QueryResult[V]]) } - def executeWithProgress[V: JsonReader]( - query: Query - )(implicit executionContext: ExecutionContext, - settings: QuerySettings = QuerySettings()): Source[QueryProgress, Future[QueryResult[V]]] = { - import QueryResult._ - val queryResult = - client.queryWithProgress(toSql(query.internalQuery)(ctx = TokenizeContext(client.serverVersion))) - queryResult.mapMaterializedValue(_.map(_.parseJson.convertTo[QueryResult[V]])) - } +// def executeWithProgress[V: JsonReader]( +// query: Query +// )(implicit executionContext: ExecutionContext, +// settings: QuerySettings = QuerySettings()): Source[QueryProgress, Future[QueryResult[V]]] = { +// import QueryResult._ +// val queryResult = +// client.queryWithProgress(toSql(query.internalQuery)(ctx = TokenizeContext(client.serverVersion))) +// queryResult.mapMaterializedValue(_.map(_.parseJson.convertTo[QueryResult[V]])) +// } override def insert[V: JsonWriter]( table: Table, From 096e1b4e275a23858ae07649d8ebbe451618c596 Mon Sep 17 00:00:00 2001 From: Leonard Wolters Date: Thu, 2 Nov 2023 10:37:45 +0100 Subject: [PATCH 05/12] Removed support for scala 2.12 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20daa0f2..e05bcfeb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - scala: [2.12.13, 2.13.8] + scala: [2.13.8] java: [temurin@11] clickhouse: [21.3, 21.8.14, 22.3] runs-on: ${{ matrix.os }} From 4056207b5d25226a7aa5b8b5bed9e4d5e6cdfc44 Mon Sep 17 00:00:00 2001 From: Leonard Wolters Date: Thu, 2 Nov 2023 10:38:54 +0100 Subject: [PATCH 06/12] Removed support for scala 2.12 --- .github/workflows/ci.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e05bcfeb..59492c6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: uses: actions/setup-java@v2 with: distribution: temurin - java-version: 11 + java-version: 17 - name: Cache sbt uses: actions/cache@v2 @@ -96,7 +96,7 @@ jobs: uses: actions/setup-java@v2 with: distribution: temurin - java-version: 11 + java-version: 17 - name: Cache sbt uses: actions/cache@v2 @@ -110,16 +110,6 @@ jobs: ~/Library/Caches/Coursier/v1 key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - - name: Download target directories (2.12.13) - uses: actions/download-artifact@v2 - with: - name: target-${{ matrix.os }}-2.12.13-${{ matrix.java }} - - - name: Inflate target directories (2.12.13) - run: | - tar xf targets.tar - rm targets.tar - - name: Download target directories (2.13.8) uses: actions/download-artifact@v2 with: From 4852f538184313ce75c428430a1c93ff5cd3e659 Mon Sep 17 00:00:00 2001 From: Leonard Wolters Date: Thu, 2 Nov 2023 10:39:31 +0100 Subject: [PATCH 07/12] Java temurin 17 --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59492c6f..188f3c00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: matrix: os: [ubuntu-latest] scala: [2.13.8] - java: [temurin@11] + java: [temurin@17] clickhouse: [21.3, 21.8.14, 22.3] runs-on: ${{ matrix.os }} steps: @@ -41,8 +41,8 @@ jobs: compose-file: './docker-compose.yml' down-flags: '--volumes' - - name: Setup Java (temurin@11) - if: matrix.java == 'temurin@11' + - name: Setup Java (temurin@17) + if: matrix.java == 'temurin@17' uses: actions/setup-java@v2 with: distribution: temurin @@ -83,7 +83,7 @@ jobs: matrix: os: [ubuntu-latest] scala: [2.13.8] - java: [temurin@11] + java: [temurin@17] runs-on: ${{ matrix.os }} steps: - name: Checkout current branch (full) @@ -91,8 +91,8 @@ jobs: with: fetch-depth: 0 - - name: Setup Java (temurin@11) - if: matrix.java == 'temurin@11' + - name: Setup Java (temurin@17) + if: matrix.java == 'temurin@17' uses: actions/setup-java@v2 with: distribution: temurin From fa24effac0c2879a721581d6b62762e0e6e16719 Mon Sep 17 00:00:00 2001 From: Leonard Wolters Date: Thu, 2 Nov 2023 11:52:18 +0100 Subject: [PATCH 08/12] Excluding Pekko for DSL --- build.sbt | 3 ++- .../dsl/execution/ClickhouseQueryExecutor.scala | 16 ++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/build.sbt b/build.sbt index 975f4c22..96dd468e 100644 --- a/build.sbt +++ b/build.sbt @@ -66,7 +66,7 @@ lazy val client: Project = (project in file("client")) "org.apache.pekko" %% "pekko-stream" % PekkoVersion, "org.apache.pekko" %% "pekko-http" % PekkoHttpVersion, "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5", - "joda-time" % "joda-time" % "2.12.2" + "joda-time" % "joda-time" % "2.12.5" ) ++ Seq("org.apache.pekko" %% "pekko-testkit" % PekkoVersion % Test) ++ Build.testDependencies.map(_ % Test) ) @@ -79,6 +79,7 @@ lazy val dsl = (project in file("dsl")) sbtrelease.ReleasePlugin.autoImport.releasePublishArtifactsAction := PgpKeys.publishSigned.value, libraryDependencies ++= Seq("com.google.guava" % "guava" % "23.0") ) + .settings(excludeDependencies ++= Seq(ExclusionRule("org.apache.pekko"))) lazy val testkit = (project in file("testkit")) .dependsOn(client) diff --git a/dsl/src/main/scala/com/crobox/clickhouse/dsl/execution/ClickhouseQueryExecutor.scala b/dsl/src/main/scala/com/crobox/clickhouse/dsl/execution/ClickhouseQueryExecutor.scala index 6c5650ad..9e42ebcc 100644 --- a/dsl/src/main/scala/com/crobox/clickhouse/dsl/execution/ClickhouseQueryExecutor.scala +++ b/dsl/src/main/scala/com/crobox/clickhouse/dsl/execution/ClickhouseQueryExecutor.scala @@ -24,14 +24,14 @@ trait ClickhouseQueryExecutor extends QueryExecutor { } // def executeWithProgress[V: JsonReader]( -// query: Query -// )(implicit executionContext: ExecutionContext, -// settings: QuerySettings = QuerySettings()): Source[QueryProgress, Future[QueryResult[V]]] = { -// import QueryResult._ -// val queryResult = -// client.queryWithProgress(toSql(query.internalQuery)(ctx = TokenizeContext(client.serverVersion))) -// queryResult.mapMaterializedValue(_.map(_.parseJson.convertTo[QueryResult[V]])) -// } + // query: Query + // )(implicit executionContext: ExecutionContext, + // settings: QuerySettings = QuerySettings()): Source[QueryProgress, Future[QueryResult[V]]] = { + // import QueryResult._ + // val queryResult = + // client.queryWithProgress(toSql(query.internalQuery)(ctx = TokenizeContext(client.serverVersion))) + // queryResult.mapMaterializedValue(_.map(_.parseJson.convertTo[QueryResult[V]])) + // } override def insert[V: JsonWriter]( table: Table, From cf9d967697f91411638dafe1c5e201689a792cf7 Mon Sep 17 00:00:00 2001 From: Leonard Wolters Date: Thu, 2 Nov 2023 11:56:58 +0100 Subject: [PATCH 09/12] Rename --- .../dsl/ClickhouseTimeSeriesIT.scala | 2 +- .../crobox/clickhouse/dsl/JoinQueryIT.scala | 24 +++++++++---------- .../com/crobox/clickhouse/dsl/QueryIT.scala | 4 ++-- .../dsl/column/AggregationFunctionsIT.scala | 12 +++++----- .../dsl/column/StringFunctionsIT.scala | 6 ++--- .../dsl/column/UUIDFunctionsIT.scala | 2 +- .../dsl/misc/QueryImprovements.scala | 24 +++++++++---------- .../com/crobox/clickhouse/dsl/package.scala | 10 ++++---- .../clickhouse/DslIntegrationSpec.scala | 2 +- 9 files changed, 43 insertions(+), 43 deletions(-) diff --git a/dsl/src/it/scala/com/crobox/clickhouse/dsl/ClickhouseTimeSeriesIT.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/ClickhouseTimeSeriesIT.scala index 4fb42a98..4090e792 100644 --- a/dsl/src/it/scala/com/crobox/clickhouse/dsl/ClickhouseTimeSeriesIT.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/ClickhouseTimeSeriesIT.scala @@ -171,7 +171,7 @@ class ClickhouseTimeSeriesIT extends DslITSpec with TableDrivenPropertyChecks { } private def getEntries(multiInterval: MultiInterval, entriesId: UUID) = - chExecutor.execute[CustomResult]( + queryExecutor.execute[CustomResult]( select(count() as "shields", toUInt64(timeSeries(timestampColumn, multiInterval)) as alias) .from(OneTestTable) .groupBy(alias) diff --git a/dsl/src/it/scala/com/crobox/clickhouse/dsl/JoinQueryIT.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/JoinQueryIT.scala index bf64a3e7..46b76ba3 100644 --- a/dsl/src/it/scala/com/crobox/clickhouse/dsl/JoinQueryIT.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/JoinQueryIT.scala @@ -11,7 +11,7 @@ class JoinQueryIT extends DslITSpec with TableDrivenPropertyChecks { .from(OneTestTable) .where(notEmpty(itemId)) .join(InnerJoin, select(itemId, col2).from(TwoTestTable).where(notEmpty(itemId))) on itemId - var resultRows = chExecutor.execute[StringResult](query).futureValue.rows + var resultRows = queryExecutor.execute[StringResult](query).futureValue.rows resultRows.length shouldBe 0 // reverse tables to check other side of ON condition @@ -19,7 +19,7 @@ class JoinQueryIT extends DslITSpec with TableDrivenPropertyChecks { .from(TwoTestTable) .where(notEmpty(itemId)) .join(InnerJoin, select(shieldId as itemId).from(OneTestTable).where(notEmpty(itemId))) on itemId - resultRows = chExecutor.execute[StringResult](query).futureValue.rows + resultRows = queryExecutor.execute[StringResult](query).futureValue.rows resultRows.length shouldBe 0 } @@ -48,7 +48,7 @@ class JoinQueryIT extends DslITSpec with TableDrivenPropertyChecks { .from(OneTestTable) .where(notEmpty(itemId)) .join(joinType, TwoTestTable) using itemId - var resultRows = chExecutor.execute[StringResult](query).futureValue.rows + var resultRows = queryExecutor.execute[StringResult](query).futureValue.rows resultRows.length shouldBe result // TABLE -- QUERY @@ -57,7 +57,7 @@ class JoinQueryIT extends DslITSpec with TableDrivenPropertyChecks { .from(OneTestTable) .where(notEmpty(itemId)) .join(joinType, select(itemId, col2).from(TwoTestTable).where(notEmpty(itemId))) using itemId - resultRows = chExecutor.execute[StringResult](query).futureValue.rows + resultRows = queryExecutor.execute[StringResult](query).futureValue.rows resultRows.length shouldBe result // QUERY -- TABLE @@ -68,7 +68,7 @@ class JoinQueryIT extends DslITSpec with TableDrivenPropertyChecks { ) .join(joinType, TwoTestTable) .where(notEmpty(itemId)) using itemId - resultRows = chExecutor.execute[StringResult](query).futureValue.rows + resultRows = queryExecutor.execute[StringResult](query).futureValue.rows resultRows.length shouldBe result // QUERY -- QUERY @@ -76,7 +76,7 @@ class JoinQueryIT extends DslITSpec with TableDrivenPropertyChecks { select(dsl.all()) .from(select(shieldId as itemId).from(OneTestTable).where(notEmpty(itemId))) .join(joinType, select(itemId, col2).from(TwoTestTable).where(notEmpty(itemId))) using itemId - resultRows = chExecutor.execute[StringResult](query).futureValue.rows + resultRows = queryExecutor.execute[StringResult](query).futureValue.rows resultRows.length shouldBe result } } @@ -97,7 +97,7 @@ class JoinQueryIT extends DslITSpec with TableDrivenPropertyChecks { .where(notEmpty(itemId)) .join(joinType, ThreeTestTable) .on((itemId, "=", itemId), (col2, "<=", col2)) - var resultRows = chExecutor.execute[StringResult](query).futureValue.rows + var resultRows = queryExecutor.execute[StringResult](query).futureValue.rows resultRows.length shouldBe result // TABLE -- QUERY @@ -106,7 +106,7 @@ class JoinQueryIT extends DslITSpec with TableDrivenPropertyChecks { .where(notEmpty(itemId)) .join(joinType, select(itemId, col2).from(ThreeTestTable).where(notEmpty(itemId))) .on((itemId, "=", itemId), (col2, "<=", col2)) - resultRows = chExecutor.execute[StringResult](query).futureValue.rows + resultRows = queryExecutor.execute[StringResult](query).futureValue.rows resultRows.length shouldBe result // QUERY -- TABLE @@ -115,7 +115,7 @@ class JoinQueryIT extends DslITSpec with TableDrivenPropertyChecks { .join(joinType, ThreeTestTable) .where(notEmpty(itemId)) .on((itemId, "=", itemId), (col2, "<=", col2)) - resultRows = chExecutor.execute[StringResult](query).futureValue.rows + resultRows = queryExecutor.execute[StringResult](query).futureValue.rows resultRows.length shouldBe result // QUERY -- QUERY @@ -124,7 +124,7 @@ class JoinQueryIT extends DslITSpec with TableDrivenPropertyChecks { .join(joinType, select(itemId, col2).from(ThreeTestTable).where(notEmpty(itemId))) .on((itemId, "=", itemId), (col2, "<=", col2)) - resultRows = chExecutor.execute[StringResult](query).futureValue.rows + resultRows = queryExecutor.execute[StringResult](query).futureValue.rows resultRows.length shouldBe result } } @@ -133,7 +133,7 @@ class JoinQueryIT extends DslITSpec with TableDrivenPropertyChecks { it should "correctly handle cross join" in { val query: OperationalQuery = select(itemId).from(select(itemId).from(TwoTestTable).join(JoinQuery.CrossJoin, ThreeTestTable)) - val resultRows = chExecutor.execute[StringResult](query).futureValue.rows + val resultRows = queryExecutor.execute[StringResult](query).futureValue.rows resultRows.length shouldBe 0 } @@ -147,7 +147,7 @@ class JoinQueryIT extends DslITSpec with TableDrivenPropertyChecks { ) .join(AllLeftJoin, ThreeTestTable) .on(itemId) - val resultRows = chExecutor.execute[StringResult](query).futureValue.rows + val resultRows = queryExecutor.execute[StringResult](query).futureValue.rows resultRows.length shouldBe 0 } } diff --git a/dsl/src/it/scala/com/crobox/clickhouse/dsl/QueryIT.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/QueryIT.scala index 2e545369..0bdf1ede 100644 --- a/dsl/src/it/scala/com/crobox/clickhouse/dsl/QueryIT.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/QueryIT.scala @@ -23,7 +23,7 @@ class QueryIT extends DslITSpec { case class Result(columnResult: String, empty: Int) implicit val resultFormat: RootJsonFormat[Result] = jsonFormat[String, Int, Result](Result.apply, "column_1", "empty") - val results: Future[QueryResult[Result]] = chExecutor.execute[Result]( + val results: Future[QueryResult[Result]] = queryExecutor.execute[Result]( select(shieldId as itemId, col1, notEmpty(col1) as "empty") from OneTestTable join (InnerJoin, TwoTestTable) using itemId ) results.futureValue.rows.map(_.columnResult) should be(table2Entries.map(_.firstColumn)) @@ -78,7 +78,7 @@ class QueryIT extends DslITSpec { } def runQry(query: OperationalQuery): Future[String] = { - val che = chExecutor.asInstanceOf[DefaultClickhouseQueryExecutor] + val che = queryExecutor.asInstanceOf[DefaultClickhouseQueryExecutor] clickhouseClient.query(che.toSql(query.internalQuery)) } } diff --git a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/AggregationFunctionsIT.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/AggregationFunctionsIT.scala index bbeddcd4..2e67ffab 100644 --- a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/AggregationFunctionsIT.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/AggregationFunctionsIT.scala @@ -22,10 +22,10 @@ class AggregationFunctionsIT extends DslITSpec { def result = columnResult.toInt } implicit val resultFormat: RootJsonFormat[Result] = jsonFormat[String, Result](Result.apply, "result") - val resultSimple = chExecutor + val resultSimple = queryExecutor .execute[Result](select(uniq(shieldId) as "result") from OneTestTable) .futureValue - val resultExact = chExecutor + val resultExact = queryExecutor .execute[Result](select(uniqExact(shieldId) as "result") from OneTestTable) .futureValue resultSimple.rows.head.result shouldBe (entries ~% delta) @@ -36,7 +36,7 @@ class AggregationFunctionsIT extends DslITSpec { it should "run quantiles" in { case class Result(result: Seq[Float]) implicit val resultFormat: RootJsonFormat[Result] = jsonFormat[Seq[Float], Result](Result.apply, "result") - val result = chExecutor + val result = queryExecutor .execute[Result]( select(quantiles(col2, 0.1F, 0.2F, 0.3F, 0.4F, 0.5F, 0.99F) as ref[Seq[Float]]("result")) from TwoTestTable ) @@ -47,7 +47,7 @@ class AggregationFunctionsIT extends DslITSpec { it should "run for each" in { case class Result(result: Seq[String]) implicit val resultFormat: RootJsonFormat[Result] = jsonFormat[Seq[String], Result](Result.apply, "result") - val result = chExecutor + val result = queryExecutor .execute[Result]( select(forEach[Int, TableColumn[Seq[Int]], Double](numbers) { column => sum(column) @@ -61,7 +61,7 @@ class AggregationFunctionsIT extends DslITSpec { it should "firstValue in aggregate" in { val resultRows = - chExecutor + queryExecutor .execute[StringResult](select(firstValue(col1) as "result").from(TwoTestTable)) .futureValue .rows @@ -71,7 +71,7 @@ class AggregationFunctionsIT extends DslITSpec { it should "lastValue in aggregate" in { val resultRows = - chExecutor + queryExecutor .execute[StringResult](select(lastValue(col1) as "result").from(TwoTestTable)) .futureValue .rows diff --git a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/StringFunctionsIT.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/StringFunctionsIT.scala index 2b1d3989..e612e754 100644 --- a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/StringFunctionsIT.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/StringFunctionsIT.scala @@ -12,7 +12,7 @@ class StringFunctionsIT extends DslITSpec { it should "split by character" in { val resultRows = - chExecutor + queryExecutor .execute[StringResult](select(arrayJoin(splitByChar(",", col1)) as "result") from TwoTestTable) .futureValue .rows @@ -22,7 +22,7 @@ class StringFunctionsIT extends DslITSpec { it should "split by string" in { val resultRows = - chExecutor + queryExecutor .execute[StringResult](select(arrayJoin(splitByString("em,", col1)) as "result") from TwoTestTable) .futureValue .rows @@ -32,7 +32,7 @@ class StringFunctionsIT extends DslITSpec { it should "concatenate string back" in { val resultRows = - chExecutor + queryExecutor .execute[StringResult](select(arrayStringConcat(splitByChar(",", col1), ",") as "result") from TwoTestTable) .futureValue .rows diff --git a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/UUIDFunctionsIT.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/UUIDFunctionsIT.scala index 78e71a95..5562616b 100644 --- a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/UUIDFunctionsIT.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/UUIDFunctionsIT.scala @@ -11,7 +11,7 @@ class UUIDFunctionsIT extends DslITSpec { it should "handle notEmpty" in { val resultRows = - chExecutor + queryExecutor .execute[StringResult](select(shieldId as "result").from(OneTestTable).where(dsl.notEmpty(shieldId))) .futureValue .rows diff --git a/dsl/src/main/scala/com/crobox/clickhouse/dsl/misc/QueryImprovements.scala b/dsl/src/main/scala/com/crobox/clickhouse/dsl/misc/QueryImprovements.scala index 77383c5d..92fd32b6 100644 --- a/dsl/src/main/scala/com/crobox/clickhouse/dsl/misc/QueryImprovements.scala +++ b/dsl/src/main/scala/com/crobox/clickhouse/dsl/misc/QueryImprovements.scala @@ -15,46 +15,46 @@ object QueryImprovements extends LazyLogging { def executeWithLogging[V: JsonReader](debug: Boolean)( implicit executionContext: ExecutionContext, - clickhouseExecutor: QueryExecutor + queryExecutor: QueryExecutor ): Future[QueryResult[V]] = { if (debug) logger.info( - s"SQL: ${tokenizer.toSql(query.internalQuery)(TokenizeContext(clickhouseExecutor.serverVersion))}" + s"SQL: ${tokenizer.toSql(query.internalQuery)(TokenizeContext(queryExecutor.serverVersion))}" ) - clickhouseExecutor.execute(query) + queryExecutor.execute(query) } def executeWithLogging[V: JsonReader](traceId: String)( implicit executionContext: ExecutionContext, - clickhouseExecutor: QueryExecutor + queryExecutor: QueryExecutor ): Future[QueryResult[V]] = { logger.info( - s"[$traceId] ${tokenizer.toSql(query.internalQuery)(TokenizeContext(clickhouseExecutor.serverVersion))}" + s"[$traceId] ${tokenizer.toSql(query.internalQuery)(TokenizeContext(queryExecutor.serverVersion))}" ) - clickhouseExecutor.execute(query) + queryExecutor.execute(query) } def executeWithLogging[V: JsonReader](traceId: Option[String])( implicit executionContext: ExecutionContext, - clickhouseExecutor: QueryExecutor + queryExecutor: QueryExecutor ): Future[QueryResult[V]] = { traceId.foreach( id => logger.info( - s"[$id] ${tokenizer.toSql(query.internalQuery)(TokenizeContext(clickhouseExecutor.serverVersion))}" + s"[$id] ${tokenizer.toSql(query.internalQuery)(TokenizeContext(queryExecutor.serverVersion))}" ) ) - clickhouseExecutor.execute(query) + queryExecutor.execute(query) } def executeWithLogging[V: JsonReader]( implicit executionContext: ExecutionContext, - clickhouseExecutor: QueryExecutor + queryExecutor: QueryExecutor ): Future[QueryResult[V]] = { logger.info( - s"SQL: ${tokenizer.toSql(query.internalQuery)(TokenizeContext(clickhouseExecutor.serverVersion))}" + s"SQL: ${tokenizer.toSql(query.internalQuery)(TokenizeContext(queryExecutor.serverVersion))}" ) - clickhouseExecutor.execute(query) + queryExecutor.execute(query) } } } diff --git a/dsl/src/main/scala/com/crobox/clickhouse/dsl/package.scala b/dsl/src/main/scala/com/crobox/clickhouse/dsl/package.scala index 48674e35..1ec71879 100644 --- a/dsl/src/main/scala/com/crobox/clickhouse/dsl/package.scala +++ b/dsl/src/main/scala/com/crobox/clickhouse/dsl/package.scala @@ -1,7 +1,7 @@ package com.crobox.clickhouse import com.crobox.clickhouse.dsl.column.ClickhouseColumnFunctions -import com.crobox.clickhouse.dsl.execution.{ClickhouseQueryExecutor, QueryExecutor, QueryResult} +import com.crobox.clickhouse.dsl.execution.{QueryExecutor, QueryResult} import com.crobox.clickhouse.dsl.marshalling.{QueryValue, QueryValueFormats} import spray.json.{JsonReader, JsonWriter} @@ -18,16 +18,16 @@ package object dsl extends ClickhouseColumnFunctions with QueryFactory with Quer def execute[V: JsonReader]( implicit executionContext: ExecutionContext, - clickhouseExecutor: QueryExecutor - ): Future[QueryResult[V]] = clickhouseExecutor.execute(query) + queryExecutor: QueryExecutor + ): Future[QueryResult[V]] = queryExecutor.execute(query) } implicit class ValueInsertion[V: JsonWriter](values: Seq[V]) { def into(table: Table)( implicit executionContext: ExecutionContext, - clickhouseExecutor: QueryExecutor - ): Future[String] = clickhouseExecutor.insert(table, values) + queryExecutor: QueryExecutor + ): Future[String] = queryExecutor.insert(table, values) } /** diff --git a/dsl/src/test/scala/com/crobox/clickhouse/DslIntegrationSpec.scala b/dsl/src/test/scala/com/crobox/clickhouse/DslIntegrationSpec.scala index 8f7bc072..da1ff0a3 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/DslIntegrationSpec.scala +++ b/dsl/src/test/scala/com/crobox/clickhouse/DslIntegrationSpec.scala @@ -30,7 +30,7 @@ trait DslIntegrationSpec implicit val ec: ExecutionContext - implicit lazy val chExecutor: QueryExecutor = ClickhouseQueryExecutor.default(clickClient) + implicit lazy val queryExecutor: QueryExecutor = ClickhouseQueryExecutor.default(clickClient) override def beforeAll(): Unit = { super.beforeAll() From 2dfde720afd53355ef55acae526825377c5daad5 Mon Sep 17 00:00:00 2001 From: Leonard Wolters Date: Thu, 2 Nov 2023 13:23:00 +0100 Subject: [PATCH 10/12] Refactored tests (corrected integration tests vs normal tests) --- build.sbt | 4 +- .../clickhouse/ClickhouseServerVersion.scala | 19 +++++ .../scala/com/crobox/clickhouse/package.scala | 21 ----- .../com/crobox/clickhouse/DslITSpec.scala | 59 ++++++++++++- .../crobox/clickhouse/dsl/JoinQueryIT.scala | 3 +- .../dsl/column/AggregationFunctionsIT.scala | 1 + .../dsl/column/ComparisonFunctionsIT.scala} | 4 +- .../dsl/column/DateTimeFunctionsIT.scala} | 4 +- .../dsl/column/EncodingFunctionsIT.scala} | 4 +- .../dsl/column/HashFunctionsIT.scala} | 4 +- .../dsl/column/HigherOrderFunctionsIT.scala} | 4 +- .../dsl/column/IPFunctionsIT.scala} | 6 +- .../dsl/column/JsonFunctionsIT.scala} | 4 +- .../column/MiscellaneousFunctionsIT.scala} | 4 +- .../dsl/column/NumericColFunctionIT.scala} | 4 +- .../dsl/column/StringFunctionsIT.scala | 78 +++++++++++++++++- .../dsl/column/TypeCastFunctionsIT.scala | 40 ++++++++- .../dsl/column/URLFunctionsIT.scala} | 4 +- .../dsl/column/UUIDFunctionsIT.scala | 1 + .../dsl/language/CollectionFunctionTest.scala | 5 +- .../clickhouse/DslIntegrationSpec.scala | 67 --------------- .../com/crobox/clickhouse/DslTestSpec.scala | 12 +-- .../com/crobox/clickhouse/TestSchema.scala | 2 +- .../dsl/column/StringFunctionsTest.scala | 82 ------------------- .../dsl/column/TypeCastFunctionsTest.scala | 46 ----------- .../language/EmptyFunctionTokenizerTest.scala | 10 +-- .../dsl/misc/RandomStringGeneratorTest.scala | 1 - 27 files changed, 233 insertions(+), 260 deletions(-) create mode 100644 client/src/main/scala/com/crobox/clickhouse/ClickhouseServerVersion.scala delete mode 100644 client/src/main/scala/com/crobox/clickhouse/package.scala rename dsl/src/{test/scala/com/crobox/clickhouse/dsl/column/ComparisonFunctionsTest.scala => it/scala/com/crobox/clickhouse/dsl/column/ComparisonFunctionsIT.scala} (86%) rename dsl/src/{test/scala/com/crobox/clickhouse/dsl/column/DateTimeFunctionsTest.scala => it/scala/com/crobox/clickhouse/dsl/column/DateTimeFunctionsIT.scala} (97%) rename dsl/src/{test/scala/com/crobox/clickhouse/dsl/column/EncodingFunctionsTest.scala => it/scala/com/crobox/clickhouse/dsl/column/EncodingFunctionsIT.scala} (82%) rename dsl/src/{test/scala/com/crobox/clickhouse/dsl/column/HashFunctionsTest.scala => it/scala/com/crobox/clickhouse/dsl/column/HashFunctionsIT.scala} (89%) rename dsl/src/{test/scala/com/crobox/clickhouse/dsl/column/HigherOrderFunctionsTest.scala => it/scala/com/crobox/clickhouse/dsl/column/HigherOrderFunctionsIT.scala} (97%) rename dsl/src/{test/scala/com/crobox/clickhouse/dsl/column/IPFunctionsTest.scala => it/scala/com/crobox/clickhouse/dsl/column/IPFunctionsIT.scala} (82%) rename dsl/src/{test/scala/com/crobox/clickhouse/dsl/column/JsonFunctionsTest.scala => it/scala/com/crobox/clickhouse/dsl/column/JsonFunctionsIT.scala} (86%) rename dsl/src/{test/scala/com/crobox/clickhouse/dsl/column/MiscellaneousFunctionsTest.scala => it/scala/com/crobox/clickhouse/dsl/column/MiscellaneousFunctionsIT.scala} (90%) rename dsl/src/{test/scala/com/crobox/clickhouse/dsl/column/NumericColFunctionTest.scala => it/scala/com/crobox/clickhouse/dsl/column/NumericColFunctionIT.scala} (96%) rename dsl/src/{test/scala/com/crobox/clickhouse/dsl/column/URLFunctionsTest.scala => it/scala/com/crobox/clickhouse/dsl/column/URLFunctionsIT.scala} (95%) rename dsl/src/{test => it}/scala/com/crobox/clickhouse/dsl/language/CollectionFunctionTest.scala (82%) delete mode 100644 dsl/src/test/scala/com/crobox/clickhouse/DslIntegrationSpec.scala delete mode 100644 dsl/src/test/scala/com/crobox/clickhouse/dsl/column/StringFunctionsTest.scala delete mode 100644 dsl/src/test/scala/com/crobox/clickhouse/dsl/column/TypeCastFunctionsTest.scala diff --git a/build.sbt b/build.sbt index 96dd468e..22392cf0 100644 --- a/build.sbt +++ b/build.sbt @@ -77,9 +77,9 @@ lazy val dsl = (project in file("dsl")) .settings( name := "dsl", sbtrelease.ReleasePlugin.autoImport.releasePublishArtifactsAction := PgpKeys.publishSigned.value, - libraryDependencies ++= Seq("com.google.guava" % "guava" % "23.0") + libraryDependencies ++= Seq("com.google.guava" % "guava" % "23.0", "com.typesafe" % "config" % "1.4.2") ) - .settings(excludeDependencies ++= Seq(ExclusionRule("org.apache.pekko"))) +// .settings(excludeDependencies ++= Seq(ExclusionRule("org.apache.pekko"))) lazy val testkit = (project in file("testkit")) .dependsOn(client) diff --git a/client/src/main/scala/com/crobox/clickhouse/ClickhouseServerVersion.scala b/client/src/main/scala/com/crobox/clickhouse/ClickhouseServerVersion.scala new file mode 100644 index 00000000..0cab5bb7 --- /dev/null +++ b/client/src/main/scala/com/crobox/clickhouse/ClickhouseServerVersion.scala @@ -0,0 +1,19 @@ +package com.crobox.clickhouse + +case class ClickhouseServerVersion(versions: Seq[Int]) { + + def minimalVersion(version: Int): Boolean = versions.head >= version + + def minimalVersion(version: Int, subVersion: Int): Boolean = + if (versions.head < version) false + else if (versions.head == version) versions(1) >= subVersion + else true +} + +object ClickhouseServerVersion { + + def apply(version: String): ClickhouseServerVersion = + ClickhouseServerVersion(version.split('.').toSeq.map(_.filter(_.isDigit)).filter(_.trim.nonEmpty).map(_.toInt)) + + def latest: ClickhouseServerVersion = ClickhouseServerVersion(versions = Seq(21, 8, 14)) +} diff --git a/client/src/main/scala/com/crobox/clickhouse/package.scala b/client/src/main/scala/com/crobox/clickhouse/package.scala deleted file mode 100644 index 833d1933..00000000 --- a/client/src/main/scala/com/crobox/clickhouse/package.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.crobox - -package object clickhouse { - case class ClickhouseServerVersion(versions: Seq[Int]) { - - def minimalVersion(version: Int): Boolean = versions.head >= version - - def minimalVersion(version: Int, subVersion: Int): Boolean = - if (versions.head < version) false - else if (versions.head == version) versions(1) >= subVersion - else true - } - - object ClickhouseServerVersion { - - def apply(version: String): ClickhouseServerVersion = - ClickhouseServerVersion(version.split('.').toSeq.map(_.filter(_.isDigit)).filter(_.trim.nonEmpty).map(_.toInt)) - - def latest: ClickhouseServerVersion = ClickhouseServerVersion(versions = Seq(21, 8, 14)) - } -} diff --git a/dsl/src/it/scala/com/crobox/clickhouse/DslITSpec.scala b/dsl/src/it/scala/com/crobox/clickhouse/DslITSpec.scala index de1116f7..58c25147 100644 --- a/dsl/src/it/scala/com/crobox/clickhouse/DslITSpec.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/DslITSpec.scala @@ -1,17 +1,28 @@ package com.crobox.clickhouse -import com.crobox.clickhouse.dsl.Query +import com.crobox.clickhouse.dsl._ +import com.crobox.clickhouse.dsl.execution.{ClickhouseQueryExecutor, QueryExecutor} +import com.crobox.clickhouse.dsl.language._ +import com.crobox.clickhouse.dsl.schemabuilder.{CreateTable, Engine} import com.crobox.clickhouse.internal.QuerySettings +import com.crobox.clickhouse.testkit.ClickhouseSpec import org.scalatest.Suite import org.scalatest.time.{Millis, Seconds, Span} -import spray.json.DefaultJsonProtocol.{jsonFormat, IntJsonFormat, StringJsonFormat} +import spray.json.DefaultJsonProtocol._ import spray.json.RootJsonFormat import scala.concurrent.Future -trait DslITSpec extends DslIntegrationSpec { +trait DslITSpec extends ClickhouseClientSpec with ClickhouseSpec with TestSchema with ClickhouseTokenizerModule { this: Suite => + implicit lazy val queryExecutor: QueryExecutor = ClickhouseQueryExecutor.default(clickClient) + implicit def ctx: TokenizeContext = TokenizeContext(clickClient.serverVersion) + + val table1Entries: Seq[Table1Entry] = Seq() + val table2Entries: Seq[Table2Entry] = Seq() + val table3Entries: Seq[Table3Entry] = Seq() + override implicit def patienceConfig: PatienceConfig = PatienceConfig(timeout = scaled(Span(10, Seconds)), interval = scaled(Span(20, Millis))) @@ -20,11 +31,53 @@ trait DslITSpec extends DslIntegrationSpec { clickClient.query(toSql(query.internalQuery, None)).map(_.trim) } + protected def r(query: Column): String = runSql(select(query)).futureValue.trim + + protected def runSql(query: OperationalQuery): Future[String] = + clickClient.query(toSql(query.internalQuery, None)) + + override def beforeAll(): Unit = { + super.beforeAll() + val tables = for { + _ <- clickClient.execute( + CreateTable(OneTestTable, Engine.Memory, ifNotExists = true).query + ) + _ <- clickClient.execute( + CreateTable( + TwoTestTable, + Engine.Memory, + ifNotExists = true + ).query + ) + _ <- clickClient.execute( + CreateTable( + ThreeTestTable, + Engine.Memory, + ifNotExists = true + ).query + ) + } yield {} + whenReady(tables) { _ => + val inserts = for { + _ <- table1Entries.into(OneTestTable) + _ <- table2Entries.into(TwoTestTable) + _ <- table3Entries.into(ThreeTestTable) + } yield {} + inserts.futureValue + } + } + + override def afterAll(): Unit = super.afterAll() +} + +object DslITSpec { case class StringResult(result: String) + implicit val stringResultFormat: RootJsonFormat[StringResult] = jsonFormat[String, StringResult](StringResult.apply, "result") case class IntResult(result: Int) + implicit val intResultFormat: RootJsonFormat[IntResult] = jsonFormat[Int, IntResult](IntResult.apply, "result") } diff --git a/dsl/src/it/scala/com/crobox/clickhouse/dsl/JoinQueryIT.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/JoinQueryIT.scala index 46b76ba3..d9a680bf 100644 --- a/dsl/src/it/scala/com/crobox/clickhouse/dsl/JoinQueryIT.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/JoinQueryIT.scala @@ -1,7 +1,8 @@ package com.crobox.clickhouse.dsl +import com.crobox.clickhouse.DslITSpec.StringResult import com.crobox.clickhouse.dsl.JoinQuery.{AllLeftJoin, InnerJoin} -import com.crobox.clickhouse.{DslITSpec, dsl} +import com.crobox.clickhouse.{dsl, DslITSpec} import org.scalatest.prop.TableDrivenPropertyChecks class JoinQueryIT extends DslITSpec with TableDrivenPropertyChecks { diff --git a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/AggregationFunctionsIT.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/AggregationFunctionsIT.scala index 2e67ffab..5189c590 100644 --- a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/AggregationFunctionsIT.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/AggregationFunctionsIT.scala @@ -1,6 +1,7 @@ package com.crobox.clickhouse.dsl.column import com.crobox.clickhouse.DslITSpec +import com.crobox.clickhouse.DslITSpec.StringResult import com.crobox.clickhouse.dsl._ import spray.json.DefaultJsonProtocol._ import spray.json.RootJsonFormat diff --git a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/ComparisonFunctionsTest.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/ComparisonFunctionsIT.scala similarity index 86% rename from dsl/src/test/scala/com/crobox/clickhouse/dsl/column/ComparisonFunctionsTest.scala rename to dsl/src/it/scala/com/crobox/clickhouse/dsl/column/ComparisonFunctionsIT.scala index ce461cba..1ea7caa7 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/ComparisonFunctionsTest.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/ComparisonFunctionsIT.scala @@ -1,9 +1,9 @@ package com.crobox.clickhouse.dsl.column -import com.crobox.clickhouse.DslIntegrationSpec +import com.crobox.clickhouse.DslITSpec import com.crobox.clickhouse.dsl._ -class ComparisonFunctionsTest extends DslIntegrationSpec { +class ComparisonFunctionsIT extends DslITSpec { "Tokenization" should "succeed for ComparisonFunctions" in { val someNum = const(10L) diff --git a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/DateTimeFunctionsTest.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/DateTimeFunctionsIT.scala similarity index 97% rename from dsl/src/test/scala/com/crobox/clickhouse/dsl/column/DateTimeFunctionsTest.scala rename to dsl/src/it/scala/com/crobox/clickhouse/dsl/column/DateTimeFunctionsIT.scala index 8a1a94c6..abedcad0 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/DateTimeFunctionsTest.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/DateTimeFunctionsIT.scala @@ -1,11 +1,11 @@ package com.crobox.clickhouse.dsl.column -import com.crobox.clickhouse.DslIntegrationSpec +import com.crobox.clickhouse.DslITSpec import com.crobox.clickhouse.TestUtils.DDTStringify import com.crobox.clickhouse.dsl._ import org.joda.time._ -class DateTimeFunctionsTest extends DslIntegrationSpec { +class DateTimeFunctionsIT extends DslITSpec { it should "succeed for DateTimeFunctions" in { val now = new DateTime().withZone(DateTimeZone.UTC) val epoch = new DateTime(0).withZone(DateTimeZone.UTC) diff --git a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/EncodingFunctionsTest.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/EncodingFunctionsIT.scala similarity index 82% rename from dsl/src/test/scala/com/crobox/clickhouse/dsl/column/EncodingFunctionsTest.scala rename to dsl/src/it/scala/com/crobox/clickhouse/dsl/column/EncodingFunctionsIT.scala index 6a422044..c114114f 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/EncodingFunctionsTest.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/EncodingFunctionsIT.scala @@ -1,11 +1,11 @@ package com.crobox.clickhouse.dsl.column -import com.crobox.clickhouse.DslIntegrationSpec +import com.crobox.clickhouse.DslITSpec import com.crobox.clickhouse.dsl._ import java.util.UUID -class EncodingFunctionsTest extends DslIntegrationSpec { +class EncodingFunctionsIT extends DslITSpec { it should "succeed for EncodingFunctions" in { r(hex(12)) shouldBe "0C" diff --git a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/HashFunctionsTest.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/HashFunctionsIT.scala similarity index 89% rename from dsl/src/test/scala/com/crobox/clickhouse/dsl/column/HashFunctionsTest.scala rename to dsl/src/it/scala/com/crobox/clickhouse/dsl/column/HashFunctionsIT.scala index ba0adfeb..926a9e01 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/HashFunctionsTest.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/HashFunctionsIT.scala @@ -1,9 +1,9 @@ package com.crobox.clickhouse.dsl.column -import com.crobox.clickhouse.DslIntegrationSpec +import com.crobox.clickhouse.DslITSpec import com.crobox.clickhouse.dsl._ -class HashFunctionsTest extends DslIntegrationSpec { +class HashFunctionsIT extends DslITSpec { it should "succeed for HashFunctions" in { val someStringData = "fooBarBaz" diff --git a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/HigherOrderFunctionsTest.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/HigherOrderFunctionsIT.scala similarity index 97% rename from dsl/src/test/scala/com/crobox/clickhouse/dsl/column/HigherOrderFunctionsTest.scala rename to dsl/src/it/scala/com/crobox/clickhouse/dsl/column/HigherOrderFunctionsIT.scala index 6511725f..46060b88 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/HigherOrderFunctionsTest.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/HigherOrderFunctionsIT.scala @@ -1,9 +1,9 @@ package com.crobox.clickhouse.dsl.column -import com.crobox.clickhouse.DslIntegrationSpec +import com.crobox.clickhouse.DslITSpec import com.crobox.clickhouse.dsl._ -class HigherOrderFunctionsTest extends DslIntegrationSpec { +class HigherOrderFunctionsIT extends DslITSpec { val arr1 = Seq(1L, 2L, 3L) it should "HigherOrderFunctions: arrayAll" in { diff --git a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/IPFunctionsTest.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/IPFunctionsIT.scala similarity index 82% rename from dsl/src/test/scala/com/crobox/clickhouse/dsl/column/IPFunctionsTest.scala rename to dsl/src/it/scala/com/crobox/clickhouse/dsl/column/IPFunctionsIT.scala index 3a3fd033..5f8ee16d 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/IPFunctionsTest.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/IPFunctionsIT.scala @@ -1,9 +1,9 @@ package com.crobox.clickhouse.dsl.column -import com.crobox.clickhouse.DslIntegrationSpec +import com.crobox.clickhouse.DslITSpec import com.crobox.clickhouse.dsl._ -class IPFunctionsTest extends DslIntegrationSpec { +class IPFunctionsIT extends DslITSpec { "Tokenization" should "succeed for IPFunctions" in { val num = toUInt32(1) @@ -13,4 +13,4 @@ class IPFunctionsTest extends DslIntegrationSpec { r(iPv6NumToString(toFixedString("0", 16))) shouldBe "3000::" r(iPv6StringToNum("3000::")) shouldBe "0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0" } -} \ No newline at end of file +} diff --git a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/JsonFunctionsTest.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/JsonFunctionsIT.scala similarity index 86% rename from dsl/src/test/scala/com/crobox/clickhouse/dsl/column/JsonFunctionsTest.scala rename to dsl/src/it/scala/com/crobox/clickhouse/dsl/column/JsonFunctionsIT.scala index 3a403af8..f23d08e2 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/JsonFunctionsTest.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/JsonFunctionsIT.scala @@ -1,9 +1,9 @@ package com.crobox.clickhouse.dsl.column -import com.crobox.clickhouse.DslIntegrationSpec +import com.crobox.clickhouse.DslITSpec import com.crobox.clickhouse.dsl._ -class JsonFunctionsTest extends DslIntegrationSpec { +class JsonFunctionsIT extends DslITSpec { it should "succeed for JsonFunctions" in { val someJson = """{"foo":"bar", "baz":123, "boz":3.1415, "bool":true}""" diff --git a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/MiscellaneousFunctionsTest.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/MiscellaneousFunctionsIT.scala similarity index 90% rename from dsl/src/test/scala/com/crobox/clickhouse/dsl/column/MiscellaneousFunctionsTest.scala rename to dsl/src/it/scala/com/crobox/clickhouse/dsl/column/MiscellaneousFunctionsIT.scala index 61e5022a..5e431576 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/MiscellaneousFunctionsTest.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/MiscellaneousFunctionsIT.scala @@ -1,9 +1,9 @@ package com.crobox.clickhouse.dsl.column import com.crobox.clickhouse.dsl._ -import com.crobox.clickhouse.{DslIntegrationSpec, dsl => CHDsl} +import com.crobox.clickhouse.{DslITSpec, dsl => CHDsl} -class MiscellaneousFunctionsTest extends DslIntegrationSpec { +class MiscellaneousFunctionsIT extends DslITSpec { it should "succeed for MiscFunctions" in { val inf = const(1) / 0 diff --git a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/NumericColFunctionTest.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/NumericColFunctionIT.scala similarity index 96% rename from dsl/src/test/scala/com/crobox/clickhouse/dsl/column/NumericColFunctionTest.scala rename to dsl/src/it/scala/com/crobox/clickhouse/dsl/column/NumericColFunctionIT.scala index f31dc6c1..0b29fd38 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/NumericColFunctionTest.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/NumericColFunctionIT.scala @@ -1,9 +1,9 @@ package com.crobox.clickhouse.dsl.column import com.crobox.clickhouse.dsl._ -import com.crobox.clickhouse.{DslIntegrationSpec, dsl => CHDsl} +import com.crobox.clickhouse.{DslITSpec, dsl => CHDsl} -class NumericColFunctionTest extends DslIntegrationSpec { +class NumericColFunctionIT extends DslITSpec { it should "succeed for ArithmeticFunctions" in { r(plus(3, 3)) shouldBe "6" diff --git a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/StringFunctionsIT.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/StringFunctionsIT.scala index e612e754..95dcdb66 100644 --- a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/StringFunctionsIT.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/StringFunctionsIT.scala @@ -1,7 +1,8 @@ package com.crobox.clickhouse.dsl.column -import com.crobox.clickhouse.DslITSpec +import com.crobox.clickhouse.DslITSpec.StringResult import com.crobox.clickhouse.dsl._ +import com.crobox.clickhouse.{dsl, DslITSpec} import java.util.UUID @@ -39,4 +40,79 @@ class StringFunctionsIT extends DslITSpec { resultRows.length shouldBe 1 resultRows.map(_.result).head shouldBe columnString } + + it should "succeed for ScalaStringFunctions" in { + r(toStringRep(pi()).contains("159")) shouldBe "1" + } + + it should "succeed for SplitMergeFunctions" in { + val theSeqOfStr = Seq("hey", "ow", "wo", "di", "ya!") + + val someStr = theSeqOfStr.mkString(",") + val someSep = "," + val someStrAsArray = "['hey','ow','wo','di','ya!']" + + r(splitByChar(someSep, someStr)) shouldBe someStrAsArray + r(splitByString(someSep, someStr)) shouldBe someStrAsArray + r(arrayStringConcat(theSeqOfStr, someSep)) shouldBe someStr + r(alphaTokens(someStr)) shouldBe someStrAsArray.filterNot(_.equals('!')) + } + + it should "succeed for StringFunctions" in { + val someStr = const("hello world") + + r(dsl.empty(someStr)) shouldBe "0" + r(notEmpty(someStr)) shouldBe "1" + r(dsl.length(someStr)) shouldBe "11" + r(lengthUTF8(someStr)) shouldBe "11" + r(lower(someStr)) shouldBe "hello world" + r(upper(someStr)) shouldBe "HELLO WORLD" + r(lowerUTF8(someStr)) shouldBe "hello world" + r(upperUTF8(someStr)) shouldBe "HELLO WORLD" + r(reverse(someStr)) shouldBe "dlrow olleh" + r(reverseUTF8(someStr)) shouldBe "dlrow olleh" + r(concat(someStr, "!")) shouldBe "hello world!" + r(substring(someStr, 2, 3)) shouldBe "ell" + r(substringUTF8(someStr, 2, 4)) shouldBe "ello" + r(appendTrailingCharIfAbsent(someStr, "!")) shouldBe "hello world!" + // r(convertCharset(someStr,"UTF16","UTF8")) shouldBe "hello world" + } + + it should "succeed for StringSearchFunctions" in { + val someStr = const("hello world") + val someNeedle = "lo" + val replace = "io" + r(position(someStr, someNeedle)) shouldBe "4" + r(positionCaseInsensitive(someStr, someNeedle)) shouldBe "4" + r(positionUTF8(someStr, someNeedle)) shouldBe "4" + r(positionUTF8CaseInsensitive(someStr, someNeedle)) shouldBe "4" + r(strMatch(someStr, someNeedle)) shouldBe "1" + r(extract(someStr, someNeedle)) shouldBe "lo" + r(extractAll(someStr, someNeedle)) shouldBe "['lo']" + r(like(someStr, someNeedle)) shouldBe "0" + r(notLike(someStr, someNeedle)) shouldBe "1" + r(replaceOne(someStr, someNeedle, replace)) shouldBe "helio world" + r(replaceAll(someStr, someNeedle, replace)) shouldBe "helio world" + r(replaceRegexpOne(someStr, someNeedle, replace)) shouldBe "helio world" + r(replaceRegexpAll(someStr, someNeedle, replace)) shouldBe "helio world" + } + + it should "keep empty as empty" in { + val query = select(All()).from(TwoTestTable).where(col1.empty()) + toSql(query.internalQuery, None) should matchSQL( + s"SELECT * FROM $database.twoTestTable WHERE empty(column_1)" + ) + } + + it should "keep notEmpty as notEmtpy" in { + var query = select(All()).from(TwoTestTable).where(col1.notEmpty()) + toSql(query.internalQuery, None) should matchSQL( + s"SELECT * FROM $database.twoTestTable WHERE notEmpty(column_1)" + ) + + query = select(All()).from(TwoTestTable).where(notEmpty(col1)) + toSql(query.internalQuery, None) should matchSQL( + s"SELECT * FROM $database.twoTestTable WHERE notEmpty(column_1)" + ) + } } diff --git a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/TypeCastFunctionsIT.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/TypeCastFunctionsIT.scala index d7c0d04f..d9a3a466 100644 --- a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/TypeCastFunctionsIT.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/TypeCastFunctionsIT.scala @@ -2,6 +2,7 @@ package com.crobox.clickhouse.dsl.column import com.crobox.clickhouse.DslITSpec import com.crobox.clickhouse.dsl._ +import com.crobox.clickhouse.dsl.schemabuilder.ColumnType class TypeCastFunctionsIT extends DslITSpec { @@ -176,6 +177,41 @@ class TypeCastFunctionsIT extends DslITSpec { r(toInt32OrDefault("12\n58 gram", 123)) should be("123") } - def r(col: TypeCastColumn[_]): String = - execute(select(col)).futureValue.trim + it should "succeed for TypeCastFunctions" in { + val someStringNum = const("123") + val someDateStr = const("2018-01-01") + val someDateTimeStr = const("2018-01-01 12:00:00") + + r(toTypeName(toUInt8(someStringNum))) shouldBe "UInt8" + r(toTypeName(toUInt16(someStringNum))) shouldBe "UInt16" + r(toTypeName(toUInt32(someStringNum))) shouldBe "UInt32" + r(toTypeName(toUInt64(someStringNum))) shouldBe "UInt64" + r(toTypeName(toInt8(someStringNum))) shouldBe "Int8" + r(toTypeName(toInt16(someStringNum))) shouldBe "Int16" + r(toTypeName(toInt32(someStringNum))) shouldBe "Int32" + r(toTypeName(toInt64(someStringNum))) shouldBe "Int64" + r(toTypeName(toFloat32(someStringNum))) shouldBe "Float32" + r(toTypeName(toFloat64(someStringNum))) shouldBe "Float64" + r(toTypeName(toUInt8OrZero(someStringNum))) shouldBe "UInt8" + r(toTypeName(toUInt16OrZero(someStringNum))) shouldBe "UInt16" + r(toTypeName(toUInt32OrZero(someStringNum))) shouldBe "UInt32" + r(toTypeName(toUInt64OrZero(someStringNum))) shouldBe "UInt64" + r(toTypeName(toInt8OrZero(someStringNum))) shouldBe "Int8" + r(toTypeName(toInt16OrZero(someStringNum))) shouldBe "Int16" + r(toTypeName(toInt32OrZero(someStringNum))) shouldBe "Int32" + r(toTypeName(toInt64OrZero(someStringNum))) shouldBe "Int64" + r(toTypeName(toFloat32OrZero(someStringNum))) shouldBe "Float32" + r(toTypeName(toFloat64OrZero(someStringNum))) shouldBe "Float64" + r(toTypeName(toDate(someDateStr))) shouldBe "Date" + r(toTypeName(toDateTime(someDateTimeStr))) shouldBe "DateTime" + r(toTypeName(toStringRep(someStringNum))) shouldBe "String" + r(toTypeName(toFixedString(someStringNum, 10))) shouldBe "FixedString(10)" + r(toTypeName(toStringCutToZero(someStringNum))) shouldBe "String" + r(reinterpret(toStringRep(65))) shouldBe "A" + r(toTypeName(cast(someStringNum, ColumnType.Int32))) shouldBe "Int32" + + r(toTypeName(toUUID(const("00000000-0000-0000-0000-000000000000")))) shouldBe "UUID" + r(toTypeName(toUUIDOrZero(const("123")))) shouldBe "UUID" + r(toTypeName(toUUIDOrNull(const("123")))) shouldBe "Nullable(UUID)" + } } diff --git a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/URLFunctionsTest.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/URLFunctionsIT.scala similarity index 95% rename from dsl/src/test/scala/com/crobox/clickhouse/dsl/column/URLFunctionsTest.scala rename to dsl/src/it/scala/com/crobox/clickhouse/dsl/column/URLFunctionsIT.scala index 5ba0038c..6fcf401f 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/URLFunctionsTest.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/URLFunctionsIT.scala @@ -1,9 +1,9 @@ package com.crobox.clickhouse.dsl.column -import com.crobox.clickhouse.DslIntegrationSpec +import com.crobox.clickhouse.DslITSpec import com.crobox.clickhouse.dsl._ -class URLFunctionsTest extends DslIntegrationSpec { +class URLFunctionsIT extends DslITSpec { it should "succeed for URLFunctions" in { val someUrl = "https://www.lib.crobox.com/clickhouse/dsl/home.html?search=true#123" diff --git a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/UUIDFunctionsIT.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/UUIDFunctionsIT.scala index 5562616b..3c4b11e5 100644 --- a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/UUIDFunctionsIT.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/UUIDFunctionsIT.scala @@ -1,5 +1,6 @@ package com.crobox.clickhouse.dsl.column +import com.crobox.clickhouse.DslITSpec.StringResult import com.crobox.clickhouse.dsl._ import com.crobox.clickhouse.{dsl, DslITSpec} diff --git a/dsl/src/test/scala/com/crobox/clickhouse/dsl/language/CollectionFunctionTest.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/language/CollectionFunctionTest.scala similarity index 82% rename from dsl/src/test/scala/com/crobox/clickhouse/dsl/language/CollectionFunctionTest.scala rename to dsl/src/it/scala/com/crobox/clickhouse/dsl/language/CollectionFunctionTest.scala index 45420398..448d4e24 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/dsl/language/CollectionFunctionTest.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/language/CollectionFunctionTest.scala @@ -1,8 +1,9 @@ package com.crobox.clickhouse.dsl.language -import com.crobox.clickhouse.DslIntegrationSpec + +import com.crobox.clickhouse.DslITSpec import com.crobox.clickhouse.dsl._ -class CollectionFunctionTest extends DslIntegrationSpec { +class CollectionFunctionTest extends DslITSpec { it should "succeed for IN functions" in { val someCollection = Seq(1, 4, 6, 9) diff --git a/dsl/src/test/scala/com/crobox/clickhouse/DslIntegrationSpec.scala b/dsl/src/test/scala/com/crobox/clickhouse/DslIntegrationSpec.scala deleted file mode 100644 index da1ff0a3..00000000 --- a/dsl/src/test/scala/com/crobox/clickhouse/DslIntegrationSpec.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.crobox.clickhouse - -import com.crobox.clickhouse.dsl._ -import com.crobox.clickhouse.dsl.execution.{ClickhouseQueryExecutor, QueryExecutor} -import com.crobox.clickhouse.dsl.language.{ClickhouseTokenizerModule, TokenizeContext} -import com.crobox.clickhouse.dsl.schemabuilder.{CreateTable, Engine} -import com.crobox.clickhouse.testkit.ClickhouseSpec -import org.scalatest.Suite - -import scala.concurrent.{ExecutionContext, Future} - -trait DslIntegrationSpec - extends ClickhouseClientSpec - with ClickhouseSpec - with TestSchema - with ClickhouseTokenizerModule { - this: Suite => - - implicit def ctx: TokenizeContext = TokenizeContext(clickClient.serverVersion) - - protected def r(query: Column): String = - runSql(select(query)).futureValue.trim - - protected def runSql(query: OperationalQuery): Future[String] = - clickClient.query(toSql(query.internalQuery, None)) - - val table1Entries: Seq[Table1Entry] = Seq() - val table2Entries: Seq[Table2Entry] = Seq() - val table3Entries: Seq[Table3Entry] = Seq() - - implicit val ec: ExecutionContext - - implicit lazy val queryExecutor: QueryExecutor = ClickhouseQueryExecutor.default(clickClient) - - override def beforeAll(): Unit = { - super.beforeAll() - val tables = for { - _ <- clickClient.execute( - CreateTable(OneTestTable, Engine.Memory, ifNotExists = true).query - ) - _ <- clickClient.execute( - CreateTable( - TwoTestTable, - Engine.Memory, - ifNotExists = true - ).query - ) - _ <- clickClient.execute( - CreateTable( - ThreeTestTable, - Engine.Memory, - ifNotExists = true - ).query - ) - } yield {} - whenReady(tables) { _ => - val inserts = for { - _ <- table1Entries.into(OneTestTable) - _ <- table2Entries.into(TwoTestTable) - _ <- table3Entries.into(ThreeTestTable) - } yield {} - inserts.futureValue - } - } - - override def afterAll(): Unit = super.afterAll() -} diff --git a/dsl/src/test/scala/com/crobox/clickhouse/DslTestSpec.scala b/dsl/src/test/scala/com/crobox/clickhouse/DslTestSpec.scala index ad69ed63..f4880d32 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/DslTestSpec.scala +++ b/dsl/src/test/scala/com/crobox/clickhouse/DslTestSpec.scala @@ -2,7 +2,7 @@ package com.crobox.clickhouse import com.crobox.clickhouse.dsl.language.{ClickhouseTokenizerModule, TokenizeContext} import com.crobox.clickhouse.dsl.{InternalQuery, OperationalQuery, TableColumn} -import com.crobox.clickhouse.testkit.ClickhouseSpec +import com.crobox.clickhouse.testkit.ClickhouseMatchers import com.typesafe.config.{Config, ConfigFactory} import org.scalatest.BeforeAndAfterAll import org.scalatest.concurrent.ScalaFutures @@ -10,17 +10,19 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers trait DslTestSpec - extends AnyFlatSpec //Like + extends AnyFlatSpec with BeforeAndAfterAll with ScalaFutures with Matchers - with ClickhouseSpec with TestSchema + with ClickhouseMatchers with ClickhouseTokenizerModule { - override val config: Config = ConfigFactory.load() + val config: Config = ConfigFactory.load() + val serverVersion: ClickhouseServerVersion = ClickhouseServerVersion(Seq(22, 3)) + override val database: String = "test" - implicit def ctx: TokenizeContext = TokenizeContext(clickClient.serverVersion) + implicit def ctx: TokenizeContext = TokenizeContext(serverVersion) def toSQL(condition: TableColumn[Boolean]): String = toSQL(Option(condition)) diff --git a/dsl/src/test/scala/com/crobox/clickhouse/TestSchema.scala b/dsl/src/test/scala/com/crobox/clickhouse/TestSchema.scala index 32c20311..008afa75 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/TestSchema.scala +++ b/dsl/src/test/scala/com/crobox/clickhouse/TestSchema.scala @@ -11,6 +11,7 @@ import java.util.UUID trait TestSchema { val database: String + private lazy val _db = database implicit object UUIDFormat extends JsonFormat[UUID] { @@ -30,7 +31,6 @@ trait TestSchema { } } - private lazy val _db = database case object OneTestTable extends Table { override lazy val database: String = _db override val name: String = "captainAmerica" diff --git a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/StringFunctionsTest.scala b/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/StringFunctionsTest.scala deleted file mode 100644 index 943131e6..00000000 --- a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/StringFunctionsTest.scala +++ /dev/null @@ -1,82 +0,0 @@ -package com.crobox.clickhouse.dsl.column - -import com.crobox.clickhouse.dsl._ -import com.crobox.clickhouse.{DslIntegrationSpec, dsl => CHDsl} - -class StringFunctionsTest extends DslIntegrationSpec { - - it should "succeed for ScalaStringFunctions" in { - r(toStringRep(pi()).contains("159")) shouldBe "1" - } - - it should "succeed for SplitMergeFunctions" in { - val theSeqOfStr = Seq("hey", "ow", "wo", "di", "ya!") - - val someStr = theSeqOfStr.mkString(",") - val someSep = "," - val someStrAsArray = "['hey','ow','wo','di','ya!']" - - r(splitByChar(someSep, someStr)) shouldBe someStrAsArray - r(splitByString(someSep, someStr)) shouldBe someStrAsArray - r(arrayStringConcat(theSeqOfStr, someSep)) shouldBe someStr - r(alphaTokens(someStr)) shouldBe someStrAsArray.filterNot(_.equals('!')) - } - - it should "succeed for StringFunctions" in { - val someStr = const("hello world") - - r(CHDsl.empty(someStr)) shouldBe "0" - r(notEmpty(someStr)) shouldBe "1" - r(CHDsl.length(someStr)) shouldBe "11" - r(lengthUTF8(someStr)) shouldBe "11" - r(lower(someStr)) shouldBe "hello world" - r(upper(someStr)) shouldBe "HELLO WORLD" - r(lowerUTF8(someStr)) shouldBe "hello world" - r(upperUTF8(someStr)) shouldBe "HELLO WORLD" - r(reverse(someStr)) shouldBe "dlrow olleh" - r(reverseUTF8(someStr)) shouldBe "dlrow olleh" - r(concat(someStr, "!")) shouldBe "hello world!" - r(substring(someStr, 2, 3)) shouldBe "ell" - r(substringUTF8(someStr, 2, 4)) shouldBe "ello" - r(appendTrailingCharIfAbsent(someStr, "!")) shouldBe "hello world!" - // r(convertCharset(someStr,"UTF16","UTF8")) shouldBe "hello world" - } - - it should "succeed for StringSearchFunctions" in { - val someStr = const("hello world") - val someNeedle = "lo" - val replace = "io" - r(position(someStr, someNeedle)) shouldBe "4" - r(positionCaseInsensitive(someStr, someNeedle)) shouldBe "4" - r(positionUTF8(someStr, someNeedle)) shouldBe "4" - r(positionUTF8CaseInsensitive(someStr, someNeedle)) shouldBe "4" - r(strMatch(someStr, someNeedle)) shouldBe "1" - r(extract(someStr, someNeedle)) shouldBe "lo" - r(extractAll(someStr, someNeedle)) shouldBe "['lo']" - r(like(someStr, someNeedle)) shouldBe "0" - r(notLike(someStr, someNeedle)) shouldBe "1" - r(replaceOne(someStr, someNeedle, replace)) shouldBe "helio world" - r(replaceAll(someStr, someNeedle, replace)) shouldBe "helio world" - r(replaceRegexpOne(someStr, someNeedle, replace)) shouldBe "helio world" - r(replaceRegexpAll(someStr, someNeedle, replace)) shouldBe "helio world" - } - - it should "keep empty as empty" in { - val query = select(All()).from(TwoTestTable).where(col1.empty()) - toSql(query.internalQuery, None) should matchSQL( - s"SELECT * FROM $database.twoTestTable WHERE empty(column_1)" - ) - } - - it should "keep notEmpty as notEmtpy" in { - var query = select(All()).from(TwoTestTable).where(col1.notEmpty()) - toSql(query.internalQuery, None) should matchSQL( - s"SELECT * FROM $database.twoTestTable WHERE notEmpty(column_1)" - ) - - query = select(All()).from(TwoTestTable).where(notEmpty(col1)) - toSql(query.internalQuery, None) should matchSQL( - s"SELECT * FROM $database.twoTestTable WHERE notEmpty(column_1)" - ) - } -} diff --git a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/TypeCastFunctionsTest.scala b/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/TypeCastFunctionsTest.scala deleted file mode 100644 index bea3eb18..00000000 --- a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/TypeCastFunctionsTest.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.crobox.clickhouse.dsl.column - -import com.crobox.clickhouse.DslIntegrationSpec -import com.crobox.clickhouse.dsl._ -import com.crobox.clickhouse.dsl.schemabuilder.ColumnType - -class TypeCastFunctionsTest extends DslIntegrationSpec { - - it should "succeed for TypeCastFunctions" in { - val someStringNum = const("123") - val someDateStr = const("2018-01-01") - val someDateTimeStr = const("2018-01-01 12:00:00") - - r(toTypeName(toUInt8(someStringNum))) shouldBe "UInt8" - r(toTypeName(toUInt16(someStringNum))) shouldBe "UInt16" - r(toTypeName(toUInt32(someStringNum))) shouldBe "UInt32" - r(toTypeName(toUInt64(someStringNum))) shouldBe "UInt64" - r(toTypeName(toInt8(someStringNum))) shouldBe "Int8" - r(toTypeName(toInt16(someStringNum))) shouldBe "Int16" - r(toTypeName(toInt32(someStringNum))) shouldBe "Int32" - r(toTypeName(toInt64(someStringNum))) shouldBe "Int64" - r(toTypeName(toFloat32(someStringNum))) shouldBe "Float32" - r(toTypeName(toFloat64(someStringNum))) shouldBe "Float64" - r(toTypeName(toUInt8OrZero(someStringNum))) shouldBe "UInt8" - r(toTypeName(toUInt16OrZero(someStringNum))) shouldBe "UInt16" - r(toTypeName(toUInt32OrZero(someStringNum))) shouldBe "UInt32" - r(toTypeName(toUInt64OrZero(someStringNum))) shouldBe "UInt64" - r(toTypeName(toInt8OrZero(someStringNum))) shouldBe "Int8" - r(toTypeName(toInt16OrZero(someStringNum))) shouldBe "Int16" - r(toTypeName(toInt32OrZero(someStringNum))) shouldBe "Int32" - r(toTypeName(toInt64OrZero(someStringNum))) shouldBe "Int64" - r(toTypeName(toFloat32OrZero(someStringNum))) shouldBe "Float32" - r(toTypeName(toFloat64OrZero(someStringNum))) shouldBe "Float64" - r(toTypeName(toDate(someDateStr))) shouldBe "Date" - r(toTypeName(toDateTime(someDateTimeStr))) shouldBe "DateTime" - r(toTypeName(toStringRep(someStringNum))) shouldBe "String" - r(toTypeName(toFixedString(someStringNum, 10))) shouldBe "FixedString(10)" - r(toTypeName(toStringCutToZero(someStringNum))) shouldBe "String" - r(reinterpret(toStringRep(65))) shouldBe "A" - r(toTypeName(cast(someStringNum, ColumnType.Int32))) shouldBe "Int32" - - r(toTypeName(toUUID(const("00000000-0000-0000-0000-000000000000")))) shouldBe "UUID" - r(toTypeName(toUUIDOrZero(const("123")))) shouldBe "UUID" - r(toTypeName(toUUIDOrNull(const("123")))) shouldBe "Nullable(UUID)" - } -} diff --git a/dsl/src/test/scala/com/crobox/clickhouse/dsl/language/EmptyFunctionTokenizerTest.scala b/dsl/src/test/scala/com/crobox/clickhouse/dsl/language/EmptyFunctionTokenizerTest.scala index 48e9b33d..ea1a5081 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/dsl/language/EmptyFunctionTokenizerTest.scala +++ b/dsl/src/test/scala/com/crobox/clickhouse/dsl/language/EmptyFunctionTokenizerTest.scala @@ -6,7 +6,7 @@ import com.crobox.clickhouse.{dsl, DslTestSpec} class EmptyFunctionTokenizerTest extends DslTestSpec { it should "UUID empty" in { - if (ClickHouseVersion.minimalVersion(21, 8)) { + if (serverVersion.minimalVersion(21, 8)) { toSQL(dsl.empty(nativeUUID)) should matchSQL("empty(uuid)") } else { toSQL(dsl.empty(nativeUUID)) should matchSQL("uuid == 0") @@ -14,7 +14,7 @@ class EmptyFunctionTokenizerTest extends DslTestSpec { } it should "UUID notEmpty" in { - if (ClickHouseVersion.minimalVersion(21, 8)) { + if (serverVersion.minimalVersion(21, 8)) { toSQL(dsl.notEmpty(nativeUUID)) should matchSQL("notEmpty(uuid)") } else { toSQL(dsl.notEmpty(nativeUUID)) should matchSQL("uuid != 0") @@ -23,7 +23,7 @@ class EmptyFunctionTokenizerTest extends DslTestSpec { it should "rewrite empty to empty(0)" in { val query = select(All()).from(TwoTestTable).where(nativeUUID.empty()) - if (ClickHouseVersion.minimalVersion(21, 8)) { + if (serverVersion.minimalVersion(21, 8)) { toSql(query.internalQuery, None) should matchSQL( s"SELECT * FROM $database.twoTestTable WHERE empty(uuid)" ) @@ -36,7 +36,7 @@ class EmptyFunctionTokenizerTest extends DslTestSpec { it should "rewrite notEmpty to notEquals(0)" in { var query = select(All()).from(TwoTestTable).where(nativeUUID.notEmpty()) - if (ClickHouseVersion.minimalVersion(21, 8)) { + if (serverVersion.minimalVersion(21, 8)) { toSql(query.internalQuery, None) should matchSQL( s"SELECT * FROM $database.twoTestTable WHERE notEmpty(uuid)" ) @@ -47,7 +47,7 @@ class EmptyFunctionTokenizerTest extends DslTestSpec { } query = select(All()).from(TwoTestTable).where(notEmpty(nativeUUID)) - if (ClickHouseVersion.minimalVersion(21, 8)) { + if (serverVersion.minimalVersion(21, 8)) { toSql(query.internalQuery, None) should matchSQL( s"SELECT * FROM $database.twoTestTable WHERE notEmpty(uuid)" ) diff --git a/dsl/src/test/scala/com/crobox/clickhouse/dsl/misc/RandomStringGeneratorTest.scala b/dsl/src/test/scala/com/crobox/clickhouse/dsl/misc/RandomStringGeneratorTest.scala index 01f6582f..033d7d46 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/dsl/misc/RandomStringGeneratorTest.scala +++ b/dsl/src/test/scala/com/crobox/clickhouse/dsl/misc/RandomStringGeneratorTest.scala @@ -13,7 +13,6 @@ class RandomStringGeneratorTest extends DslTestSpec { it should "avoid doubles of equal length" in { val n = 1000000 -// (1 to n).map(i => RandomStringGenerator.random(6)).toSet should have size(n) (1 to n).map(i => RandomStringGenerator.random(6)).toSet.size should be(n +- 50) } } From 514f5775857a26f7e5dcdb1b850e70343378c663 Mon Sep 17 00:00:00 2001 From: Leonard Wolters Date: Thu, 2 Nov 2023 13:29:10 +0100 Subject: [PATCH 11/12] Fixed tests --- .../dsl/column/AggregationFunctionsIT.scala | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/AggregationFunctionsIT.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/AggregationFunctionsIT.scala index 5189c590..d87e9659 100644 --- a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/AggregationFunctionsIT.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/AggregationFunctionsIT.scala @@ -1,21 +1,22 @@ package com.crobox.clickhouse.dsl.column import com.crobox.clickhouse.DslITSpec -import com.crobox.clickhouse.DslITSpec.StringResult +import com.crobox.clickhouse.DslITSpec.IntResult import com.crobox.clickhouse.dsl._ +import com.typesafe.scalalogging.LazyLogging import spray.json.DefaultJsonProtocol._ import spray.json.RootJsonFormat import java.util.UUID -class AggregationFunctionsIT extends DslITSpec { +class AggregationFunctionsIT extends DslITSpec with LazyLogging { private val entries = 200145 private val delta = 2 override val table1Entries: Seq[Table1Entry] = Seq.fill(entries)(Table1Entry(UUID.randomUUID(), numbers = Seq(1, 2, 3))) override val table2Entries: Seq[Table2Entry] = { - (1 to entries).map(i => Table2Entry(UUID.randomUUID(), i + "_" + randomString, randomInt, randomString, None)) + (1 to entries).map(i => Table2Entry(UUID.randomUUID(), randomString, i, randomString, None)) } "Combinators" should "apply for aggregations" in { @@ -63,20 +64,22 @@ class AggregationFunctionsIT extends DslITSpec { it should "firstValue in aggregate" in { val resultRows = queryExecutor - .execute[StringResult](select(firstValue(col1) as "result").from(TwoTestTable)) + .execute[IntResult](select(firstValue(col2) as "result").from(TwoTestTable)) .futureValue .rows resultRows.length shouldBe 1 - resultRows.map(_.result).head.startsWith("1_") should be(true) + logger.info("FIRST_VALUE: " + resultRows.map(_.result).head) + resultRows.map(_.result).head should be(1) } it should "lastValue in aggregate" in { val resultRows = queryExecutor - .execute[StringResult](select(lastValue(col1) as "result").from(TwoTestTable)) + .execute[IntResult](select(lastValue(col2) as "result").from(TwoTestTable)) .futureValue .rows resultRows.length shouldBe 1 - resultRows.map(_.result).head.startsWith(s"${entries}_") should be(true) + logger.info("LAST_VALUE: " + resultRows.map(_.result).head) + resultRows.map(_.result).head should be(entries) } } From 5f52fcb9ed2173cdd10cd4be6ead88874f6d3743 Mon Sep 17 00:00:00 2001 From: Leonard Wolters Date: Thu, 2 Nov 2023 13:34:00 +0100 Subject: [PATCH 12/12] Fixed tests --- .../clickhouse/dsl/column/AggregationFunctionsIT.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/AggregationFunctionsIT.scala b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/AggregationFunctionsIT.scala index d87e9659..99648966 100644 --- a/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/AggregationFunctionsIT.scala +++ b/dsl/src/it/scala/com/crobox/clickhouse/dsl/column/AggregationFunctionsIT.scala @@ -64,22 +64,24 @@ class AggregationFunctionsIT extends DslITSpec with LazyLogging { it should "firstValue in aggregate" in { val resultRows = queryExecutor - .execute[IntResult](select(firstValue(col2) as "result").from(TwoTestTable)) + .execute[IntResult]( + select(firstValue(col2) as "result").from(select(col2).from(TwoTestTable).orderBy(col2)) + ) .futureValue .rows resultRows.length shouldBe 1 - logger.info("FIRST_VALUE: " + resultRows.map(_.result).head) resultRows.map(_.result).head should be(1) } it should "lastValue in aggregate" in { val resultRows = queryExecutor - .execute[IntResult](select(lastValue(col2) as "result").from(TwoTestTable)) + .execute[IntResult]( + select(lastValue(col2) as "result").from(select(col2).from(TwoTestTable).orderBy(col2)) + ) .futureValue .rows resultRows.length shouldBe 1 - logger.info("LAST_VALUE: " + resultRows.map(_.result).head) resultRows.map(_.result).head should be(entries) } }