From ceaf17faff29b9c5b6dd74ee2af8b479b3d599c3 Mon Sep 17 00:00:00 2001 From: Leonard Wolters Date: Tue, 28 May 2024 16:19:32 +0200 Subject: [PATCH] Small improvements that helps debugging SQL query differences --- .../language/ClickhouseTokenizerModule.scala | 6 +- .../dsl/column/ArrayFunctionsTest.scala | 28 +++--- .../language/ClickhouseTokenizerTest.scala | 89 +++++++++++-------- 3 files changed, 71 insertions(+), 52 deletions(-) diff --git a/dsl/src/main/scala/com/crobox/clickhouse/dsl/language/ClickhouseTokenizerModule.scala b/dsl/src/main/scala/com/crobox/clickhouse/dsl/language/ClickhouseTokenizerModule.scala index 33c69d53..61d8f612 100644 --- a/dsl/src/main/scala/com/crobox/clickhouse/dsl/language/ClickhouseTokenizerModule.scala +++ b/dsl/src/main/scala/com/crobox/clickhouse/dsl/language/ClickhouseTokenizerModule.scala @@ -16,7 +16,7 @@ case class TokenizeContext( var tableAliases: Map[Table, String] = Map.empty, var useTableAlias: Boolean = false, fDelim: String = ", ", // function delimiter - vDelim: String = "," // values delimiter + vDelim: String = ", " // values delimiter ) { def incrementJoinNumber(): Unit = joinNr += 1 @@ -80,7 +80,7 @@ trait ClickhouseTokenizerModule } protected def tokenizeSeqCol(columns: Column*)(implicit ctx: TokenizeContext): String = - columns.map(tokenizeColumn).mkString(",") + columns.map(tokenizeColumn).mkString(ctx.vDelim) override def toSql(query: InternalQuery, formatting: Option[String] = Some("JSON"))(implicit ctx: TokenizeContext @@ -151,7 +151,7 @@ trait ClickhouseTokenizerModule case alias: AliasedColumn[_] => val originalColumnToken = tokenizeColumn(alias.original) if (originalColumnToken.isEmpty) alias.quoted else s"$originalColumnToken AS ${alias.quoted}" - case tuple: TupleColumn[_] => s"(${tuple.elements.map(tokenizeColumn).mkString(",")})" + case tuple: TupleColumn[_] => s"(${tuple.elements.map(tokenizeColumn).mkString(ctx.vDelim)})" case col: ExpressionColumn[_] => tokenizeExpressionColumn(col) case col: Column => col.quoted } diff --git a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/ArrayFunctionsTest.scala b/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/ArrayFunctionsTest.scala index d94b28d8..cedf3de0 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/ArrayFunctionsTest.scala +++ b/dsl/src/test/scala/com/crobox/clickhouse/dsl/column/ArrayFunctionsTest.scala @@ -8,53 +8,53 @@ class ArrayFunctionsTest extends DslTestSpec { val arrayNumbersSerialized = "[1, 2, 3, 4]" it should "arrayFunction: array" in { - toSQL(select(Array())) should be("SELECT []") - toSQL(select(Array(1, 2))) should be("SELECT [1, 2]") + toSQL(select(Array())) should matchSQL("SELECT []") + toSQL(select(Array(1, 2))) should matchSQL("SELECT [1, 2]") } it should "arrayFunction: arrayConcat" in { - toSQL(select(arrayConcat(Array(1)))) should be("SELECT arrayConcat([1])") - toSQL(select(arrayConcat(Array(1), Array(2), Array(3)))) should be("SELECT arrayConcat([1], [2], [3])") + toSQL(select(arrayConcat(Array(1)))) should matchSQL("SELECT arrayConcat([1])") + toSQL(select(arrayConcat(Array(1), Array(2), Array(3)))) should matchSQL("SELECT arrayConcat([1], [2], [3])") } it should "arrayFunction: has" in { var query = select(All()).from(OneTestTable).where(has(numbers, 1)) - toSQL(query) should be("WHERE has(numbers, 1)") + toSQL(query) should matchSQL("WHERE has(numbers, 1)") query = select(All()).from(OneTestTable).where(has(arrayNumbers, 1)) - toSQL(query) should be(s"WHERE has($arrayNumbersSerialized, 1)") + toSQL(query) should matchSQL(s"WHERE has($arrayNumbersSerialized, 1)") } it should "arrayFunction: hasAll" in { var query = select(All()).from(OneTestTable).where(hasAll(numbers, Array(1, 2))) - toSQL(query) should be("WHERE hasAll(numbers, [1, 2])") + toSQL(query) should matchSQL("WHERE hasAll(numbers, [1, 2])") query = select(All()).from(OneTestTable).where(hasAll(arrayNumbers, Array(1, 2))) - toSQL(query) should be(s"WHERE hasAll($arrayNumbersSerialized, [1, 2])") + toSQL(query) should matchSQL(s"WHERE hasAll($arrayNumbersSerialized, [1, 2])") } it should "arrayFunction: hasAny" in { var query = select(All()).from(OneTestTable).where(hasAny(numbers, Array(1, 2))) - toSQL(query) should be("WHERE hasAny(numbers, [1, 2])") + toSQL(query) should matchSQL("WHERE hasAny(numbers, [1, 2])") query = select(All()).from(OneTestTable).where(hasAny(arrayNumbers, Array(1, 2))) - toSQL(query) should be(s"WHERE hasAny($arrayNumbersSerialized, [1, 2])") + toSQL(query) should matchSQL(s"WHERE hasAny($arrayNumbersSerialized, [1, 2])") } it should "arrayFunction: resize" in { var query = select(arrayResize(numbers, 4, 0)) - toSQL(query) should be("SELECT arrayResize(numbers,4,0)") + toSQL(query) should matchSQL("SELECT arrayResize(numbers,4,0)") query = select(arrayResize(arrayNumbers, 4, 0)) - toSQL(query) should be(s"SELECT arrayResize($arrayNumbersSerialized,4,0)") + toSQL(query) should matchSQL(s"SELECT arrayResize($arrayNumbersSerialized,4,0)") } it should "arrayFunction: join" in { - toSQL(select(arrayJoin(Array(shieldId, itemId)))) should be(s"SELECT arrayJoin([shield_id, item_id])") + toSQL(select(arrayJoin(Array(shieldId, itemId)))) should matchSQL(s"SELECT arrayJoin([shield_id, item_id])") } it should "arrayFunction: join with concat" in { val col = arrayConcat(Array(shieldId, itemId), Array[String]()) - toSQL(select(arrayJoin(col))) should be(s"SELECT arrayJoin(arrayConcat([shield_id, item_id], []))") + toSQL(select(arrayJoin(col))) should matchSQL(s"SELECT arrayJoin(arrayConcat([shield_id, item_id], []))") } } diff --git a/dsl/src/test/scala/com/crobox/clickhouse/dsl/language/ClickhouseTokenizerTest.scala b/dsl/src/test/scala/com/crobox/clickhouse/dsl/language/ClickhouseTokenizerTest.scala index dee319b4..2c005608 100644 --- a/dsl/src/test/scala/com/crobox/clickhouse/dsl/language/ClickhouseTokenizerTest.scala +++ b/dsl/src/test/scala/com/crobox/clickhouse/dsl/language/ClickhouseTokenizerTest.scala @@ -38,9 +38,11 @@ class ClickhouseTokenizerTest extends DslTestSpec { it should "add simple condition between columns" in { val select = SelectQuery(Seq(shieldId)) val query = toSql( - InternalQuery(Some(select), - Some(TableFromQuery[OneTestTable.type](OneTestTable)), - where = Some(shieldId < itemId)) + InternalQuery( + Some(select), + Some(TableFromQuery[OneTestTable.type](OneTestTable)), + where = Some(shieldId < itemId) + ) ) query should be(s"SELECT shield_id FROM ${OneTestTable.quoted} WHERE shield_id < item_id FORMAT JSON") } @@ -50,9 +52,11 @@ class ClickhouseTokenizerTest extends DslTestSpec { val uuid = UUID.randomUUID() val query = toSql( - InternalQuery(Some(select), - Some(TableFromQuery[OneTestTable.type](OneTestTable)), - where = Some(shieldId < uuid)) + InternalQuery( + Some(select), + Some(TableFromQuery[OneTestTable.type](OneTestTable)), + where = Some(shieldId < uuid) + ) ) query should be(s"SELECT shield_id FROM ${OneTestTable.quoted} WHERE shield_id < '$uuid' FORMAT JSON") } @@ -61,9 +65,11 @@ class ClickhouseTokenizerTest extends DslTestSpec { val select = SelectQuery(Seq(shieldId)) val uuid = UUID.randomUUID() val query = toSql( - InternalQuery(Some(select), - Some(TableFromQuery[OneTestTable.type](OneTestTable)), - where = Some(shieldId < uuid and shieldId < itemId)) + InternalQuery( + Some(select), + Some(TableFromQuery[OneTestTable.type](OneTestTable)), + where = Some(shieldId < uuid and shieldId < itemId) + ) ) query should be( s"SELECT shield_id FROM ${OneTestTable.quoted} WHERE shield_id < '$uuid' AND shield_id < item_id FORMAT JSON" @@ -74,9 +80,11 @@ class ClickhouseTokenizerTest extends DslTestSpec { val alias = shieldId as "preferable" val select = SelectQuery(Seq(alias)) val query = toSql( - InternalQuery(Some(select), - Some(TableFromQuery[OneTestTable.type](OneTestTable)), - groupBy = Some(GroupByQuery(Seq(alias)))) + InternalQuery( + Some(select), + Some(TableFromQuery[OneTestTable.type](OneTestTable)), + groupBy = Some(GroupByQuery(Seq(alias))) + ) ) query should be(s"SELECT shield_id AS preferable FROM ${OneTestTable.quoted} GROUP BY preferable FORMAT JSON") } @@ -98,9 +106,11 @@ class ClickhouseTokenizerTest extends DslTestSpec { it should "group by with cube if using group by mode" in { val select = SelectQuery(Seq(shieldId)) val query = toSql( - InternalQuery(Some(select), - Some(TableFromQuery[OneTestTable.type](OneTestTable)), - groupBy = Some(GroupByQuery(mode = Some(GroupByQuery.WithCube)))) + InternalQuery( + Some(select), + Some(TableFromQuery[OneTestTable.type](OneTestTable)), + groupBy = Some(GroupByQuery(mode = Some(GroupByQuery.WithCube))) + ) ) query should be(s"SELECT shield_id FROM ${OneTestTable.quoted} WITH CUBE FORMAT JSON") } @@ -166,7 +176,7 @@ class ClickhouseTokenizerTest extends DslTestSpec { it should "generate CONDITIONAL cases" in { this.tokenizeColumn(switch(const(3))) shouldBe "3" this.tokenizeColumn(switch(shieldId, columnCase(col1.isEq("test"), itemId))) shouldBe - (s"CASE WHEN ${col1.name} = 'test' THEN ${itemId.name} ELSE ${shieldId.name} END") + s"CASE WHEN ${col1.name} = 'test' THEN ${itemId.name} ELSE ${shieldId.name} END" } it should "generate CONDITIONAL multiIf" in { @@ -176,13 +186,13 @@ class ClickhouseTokenizerTest extends DslTestSpec { // test single case this.tokenizeColumn(multiIf(shieldId, columnCase(col1.isEq("test"), itemId))) shouldBe - (s"if(${col1.name} = 'test', ${itemId.name}, ${shieldId.name})") + s"if(${col1.name} = 'test', ${itemId.name}, ${shieldId.name})" // test multi cases this.tokenizeColumn( multiIf(shieldId, columnCase(col1.isEq("test"), itemId), columnCase(col1.isEq("test"), itemId)) ) shouldBe - (s"multiIf(${col1.name} = 'test', ${itemId.name}, ${col1.name} = 'test', ${itemId.name}, ${shieldId.name})") + s"multiIf(${col1.name} = 'test', ${itemId.name}, ${col1.name} = 'test', ${itemId.name}, ${shieldId.name})" } it should "use constant" in { @@ -193,36 +203,45 @@ class ClickhouseTokenizerTest extends DslTestSpec { val col = RawColumn("Robert'); DROP TABLE students;") val select = SelectQuery(Seq(col)) val query = toSql(InternalQuery(select = Some(select), where = Some(col))) - query should be(s"SELECT ${col.rawSql} WHERE ${col.rawSql} FORMAT JSON") + query should matchSQL(s"SELECT ${col.rawSql} WHERE ${col.rawSql} FORMAT JSON") } it should "build with combinators" in { - this.tokenizeColumn(CombinedAggregatedFunction(Combinator.If(col1.isEq("test")), uniq(col1))) shouldBe s"uniqIf(${col1.name},${col1.name} = 'test')" - this.tokenizeColumn(CombinedAggregatedFunction(Combinator.If(col1.isEq("test")), uniqHLL12(col1))) shouldBe s"uniqHLL12If(${col1.name},${col1.name} = 'test')" - this.tokenizeColumn(CombinedAggregatedFunction(Combinator.If(col1.isEq("test")), uniqCombined(col1))) shouldBe s"uniqCombinedIf(${col1.name},${col1.name} = 'test')" + this.tokenizeColumn(CombinedAggregatedFunction(Combinator.If(col1.isEq("test")), uniq(col1))) should matchSQL( + s"uniqIf(${col1.name}, ${col1.name} = 'test')" + ) + this.tokenizeColumn(CombinedAggregatedFunction(Combinator.If(col1.isEq("test")), uniqHLL12(col1))) should matchSQL( + s"uniqHLL12If(${col1.name}, ${col1.name} = 'test')" + ) + this.tokenizeColumn( + CombinedAggregatedFunction(Combinator.If(col1.isEq("test")), uniqCombined(col1)) + ) should matchSQL(s"uniqCombinedIf(${col1.name}, ${col1.name} = 'test')") this.tokenizeColumn( - CombinedAggregatedFunction(Combinator.If(col1.isEq("test")), - CombinedAggregatedFunction(Combinator.If(col2.isEq(3)), uniqExact(col1))) - ) shouldBe s"uniqExactIfIf(${col1.name},${col2.name} = 3,${col1.name} = 'test')" + CombinedAggregatedFunction( + Combinator.If(col1.isEq("test")), + CombinedAggregatedFunction(Combinator.If(col2.isEq(3)), uniqExact(col1)) + ) + ) should matchSQL(s"uniqExactIfIf(${col1.name}, ${col2.name} = 3, ${col1.name} = 'test')") } it should "uniq for multiple columns" in { - this.tokenizeColumn(uniq(col1, col2)) shouldBe s"uniq(${col1.name},${col2.name})" - this.tokenizeColumn(uniqHLL12(col1, col2)) shouldBe s"uniqHLL12(${col1.name},${col2.name})" - this.tokenizeColumn(uniqExact(col1, col2)) shouldBe s"uniqExact(${col1.name},${col2.name})" - this.tokenizeColumn(uniqCombined(col1, col2)) shouldBe s"uniqCombined(${col1.name},${col2.name})" - + this.tokenizeColumn(uniq(col1, col2)) should matchSQL(s"uniq(${col1.name}, ${col2.name})") + this.tokenizeColumn(uniqHLL12(col1, col2)) should matchSQL(s"uniqHLL12(${col1.name}, ${col2.name})") + this.tokenizeColumn(uniqExact(col1, col2)) should matchSQL(s"uniqExact(${col1.name}, ${col2.name})") + this.tokenizeColumn(uniqCombined(col1, col2)) should matchSQL(s"uniqCombined(${col1.name}, ${col2.name})") } it should "use zone name for monthly" in { this.tokenizeTimeSeries( TimeSeries( timestampColumn, - MultiInterval(DateTime.now(DateTimeZone.forOffsetHours(2)), - DateTime.now(DateTimeZone.forOffsetHours(2)), - MultiDuration(TimeUnit.Month)) + MultiInterval( + DateTime.now(DateTimeZone.forOffsetHours(2)), + DateTime.now(DateTimeZone.forOffsetHours(2)), + MultiDuration(TimeUnit.Month) + ) ) - ) shouldBe "toDateTime(toStartOfMonth(toDateTime(ts / 1000), 'Etc/GMT-2'), 'Etc/GMT-2')" + ) should matchSQL("toDateTime(toStartOfMonth(toDateTime(ts / 1000), 'Etc/GMT-2'), 'Etc/GMT-2')") } it should "quote them correctly" in { @@ -230,6 +249,6 @@ class ClickhouseTokenizerTest extends DslTestSpec { val col = RefColumn(name) val select = SelectQuery(Seq(col)) val query = toSql(InternalQuery(select = Some(select))) - query should be(s"SELECT `$name` FORMAT JSON") + query should matchSQL(s"SELECT `$name` FORMAT JSON") } }