Skip to content

Commit

Permalink
Small improvements that helps debugging SQL query differences
Browse files Browse the repository at this point in the history
  • Loading branch information
Leonard Wolters committed May 28, 2024
1 parent 5d8c0ad commit ceaf17f
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 52 deletions.
6 changes: 3 additions & 3 deletions dsl/src/main/scala/com/crobox/clickhouse/dsl/language/ClickhouseTokenizerModule.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ case class TokenizeContext(
var tableAliases: Map[Table, String] = Map.empty,
var useTableAlias: Boolean = false,
fDelim: String = ", ", // function delimiter
vDelim: String = "," // values delimiter
vDelim: String = ", " // values delimiter
) {

def incrementJoinNumber(): Unit = joinNr += 1
Expand Down Expand Up @@ -80,7 +80,7 @@ trait ClickhouseTokenizerModule
}

protected def tokenizeSeqCol(columns: Column*)(implicit ctx: TokenizeContext): String =
columns.map(tokenizeColumn).mkString(",")
columns.map(tokenizeColumn).mkString(ctx.vDelim)

override def toSql(query: InternalQuery, formatting: Option[String] = Some("JSON"))(implicit
ctx: TokenizeContext
Expand Down Expand Up @@ -151,7 +151,7 @@ trait ClickhouseTokenizerModule
case alias: AliasedColumn[_] =>
val originalColumnToken = tokenizeColumn(alias.original)
if (originalColumnToken.isEmpty) alias.quoted else s"$originalColumnToken AS ${alias.quoted}"
case tuple: TupleColumn[_] => s"(${tuple.elements.map(tokenizeColumn).mkString(",")})"
case tuple: TupleColumn[_] => s"(${tuple.elements.map(tokenizeColumn).mkString(ctx.vDelim)})"
case col: ExpressionColumn[_] => tokenizeExpressionColumn(col)
case col: Column => col.quoted
}
Expand Down
28 changes: 14 additions & 14 deletions dsl/src/test/scala/com/crobox/clickhouse/dsl/column/ArrayFunctionsTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,53 +8,53 @@ class ArrayFunctionsTest extends DslTestSpec {
val arrayNumbersSerialized = "[1, 2, 3, 4]"

it should "arrayFunction: array" in {
toSQL(select(Array())) should be("SELECT []")
toSQL(select(Array(1, 2))) should be("SELECT [1, 2]")
toSQL(select(Array())) should matchSQL("SELECT []")
toSQL(select(Array(1, 2))) should matchSQL("SELECT [1, 2]")
}

it should "arrayFunction: arrayConcat" in {
toSQL(select(arrayConcat(Array(1)))) should be("SELECT arrayConcat([1])")
toSQL(select(arrayConcat(Array(1), Array(2), Array(3)))) should be("SELECT arrayConcat([1], [2], [3])")
toSQL(select(arrayConcat(Array(1)))) should matchSQL("SELECT arrayConcat([1])")
toSQL(select(arrayConcat(Array(1), Array(2), Array(3)))) should matchSQL("SELECT arrayConcat([1], [2], [3])")
}

it should "arrayFunction: has" in {
var query = select(All()).from(OneTestTable).where(has(numbers, 1))
toSQL(query) should be("WHERE has(numbers, 1)")
toSQL(query) should matchSQL("WHERE has(numbers, 1)")

query = select(All()).from(OneTestTable).where(has(arrayNumbers, 1))
toSQL(query) should be(s"WHERE has($arrayNumbersSerialized, 1)")
toSQL(query) should matchSQL(s"WHERE has($arrayNumbersSerialized, 1)")
}

it should "arrayFunction: hasAll" in {
var query = select(All()).from(OneTestTable).where(hasAll(numbers, Array(1, 2)))
toSQL(query) should be("WHERE hasAll(numbers, [1, 2])")
toSQL(query) should matchSQL("WHERE hasAll(numbers, [1, 2])")

query = select(All()).from(OneTestTable).where(hasAll(arrayNumbers, Array(1, 2)))
toSQL(query) should be(s"WHERE hasAll($arrayNumbersSerialized, [1, 2])")
toSQL(query) should matchSQL(s"WHERE hasAll($arrayNumbersSerialized, [1, 2])")
}

it should "arrayFunction: hasAny" in {
var query = select(All()).from(OneTestTable).where(hasAny(numbers, Array(1, 2)))
toSQL(query) should be("WHERE hasAny(numbers, [1, 2])")
toSQL(query) should matchSQL("WHERE hasAny(numbers, [1, 2])")

query = select(All()).from(OneTestTable).where(hasAny(arrayNumbers, Array(1, 2)))
toSQL(query) should be(s"WHERE hasAny($arrayNumbersSerialized, [1, 2])")
toSQL(query) should matchSQL(s"WHERE hasAny($arrayNumbersSerialized, [1, 2])")
}

it should "arrayFunction: resize" in {
var query = select(arrayResize(numbers, 4, 0))
toSQL(query) should be("SELECT arrayResize(numbers,4,0)")
toSQL(query) should matchSQL("SELECT arrayResize(numbers,4,0)")

query = select(arrayResize(arrayNumbers, 4, 0))
toSQL(query) should be(s"SELECT arrayResize($arrayNumbersSerialized,4,0)")
toSQL(query) should matchSQL(s"SELECT arrayResize($arrayNumbersSerialized,4,0)")
}

it should "arrayFunction: join" in {
toSQL(select(arrayJoin(Array(shieldId, itemId)))) should be(s"SELECT arrayJoin([shield_id, item_id])")
toSQL(select(arrayJoin(Array(shieldId, itemId)))) should matchSQL(s"SELECT arrayJoin([shield_id, item_id])")
}

it should "arrayFunction: join with concat" in {
val col = arrayConcat(Array(shieldId, itemId), Array[String]())
toSQL(select(arrayJoin(col))) should be(s"SELECT arrayJoin(arrayConcat([shield_id, item_id], []))")
toSQL(select(arrayJoin(col))) should matchSQL(s"SELECT arrayJoin(arrayConcat([shield_id, item_id], []))")
}
}
89 changes: 54 additions & 35 deletions dsl/src/test/scala/com/crobox/clickhouse/dsl/language/ClickhouseTokenizerTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,11 @@ class ClickhouseTokenizerTest extends DslTestSpec {
it should "add simple condition between columns" in {
val select = SelectQuery(Seq(shieldId))
val query = toSql(
InternalQuery(Some(select),
Some(TableFromQuery[OneTestTable.type](OneTestTable)),
where = Some(shieldId < itemId))
InternalQuery(
Some(select),
Some(TableFromQuery[OneTestTable.type](OneTestTable)),
where = Some(shieldId < itemId)
)
)
query should be(s"SELECT shield_id FROM ${OneTestTable.quoted} WHERE shield_id < item_id FORMAT JSON")
}
Expand All @@ -50,9 +52,11 @@ class ClickhouseTokenizerTest extends DslTestSpec {
val uuid = UUID.randomUUID()
val query =
toSql(
InternalQuery(Some(select),
Some(TableFromQuery[OneTestTable.type](OneTestTable)),
where = Some(shieldId < uuid))
InternalQuery(
Some(select),
Some(TableFromQuery[OneTestTable.type](OneTestTable)),
where = Some(shieldId < uuid)
)
)
query should be(s"SELECT shield_id FROM ${OneTestTable.quoted} WHERE shield_id < '$uuid' FORMAT JSON")
}
Expand All @@ -61,9 +65,11 @@ class ClickhouseTokenizerTest extends DslTestSpec {
val select = SelectQuery(Seq(shieldId))
val uuid = UUID.randomUUID()
val query = toSql(
InternalQuery(Some(select),
Some(TableFromQuery[OneTestTable.type](OneTestTable)),
where = Some(shieldId < uuid and shieldId < itemId))
InternalQuery(
Some(select),
Some(TableFromQuery[OneTestTable.type](OneTestTable)),
where = Some(shieldId < uuid and shieldId < itemId)
)
)
query should be(
s"SELECT shield_id FROM ${OneTestTable.quoted} WHERE shield_id < '$uuid' AND shield_id < item_id FORMAT JSON"
Expand All @@ -74,9 +80,11 @@ class ClickhouseTokenizerTest extends DslTestSpec {
val alias = shieldId as "preferable"
val select = SelectQuery(Seq(alias))
val query = toSql(
InternalQuery(Some(select),
Some(TableFromQuery[OneTestTable.type](OneTestTable)),
groupBy = Some(GroupByQuery(Seq(alias))))
InternalQuery(
Some(select),
Some(TableFromQuery[OneTestTable.type](OneTestTable)),
groupBy = Some(GroupByQuery(Seq(alias)))
)
)
query should be(s"SELECT shield_id AS preferable FROM ${OneTestTable.quoted} GROUP BY preferable FORMAT JSON")
}
Expand All @@ -98,9 +106,11 @@ class ClickhouseTokenizerTest extends DslTestSpec {
it should "group by with cube if using group by mode" in {
val select = SelectQuery(Seq(shieldId))
val query = toSql(
InternalQuery(Some(select),
Some(TableFromQuery[OneTestTable.type](OneTestTable)),
groupBy = Some(GroupByQuery(mode = Some(GroupByQuery.WithCube))))
InternalQuery(
Some(select),
Some(TableFromQuery[OneTestTable.type](OneTestTable)),
groupBy = Some(GroupByQuery(mode = Some(GroupByQuery.WithCube)))
)
)
query should be(s"SELECT shield_id FROM ${OneTestTable.quoted} WITH CUBE FORMAT JSON")
}
Expand Down Expand Up @@ -166,7 +176,7 @@ class ClickhouseTokenizerTest extends DslTestSpec {
it should "generate CONDITIONAL cases" in {
this.tokenizeColumn(switch(const(3))) shouldBe "3"
this.tokenizeColumn(switch(shieldId, columnCase(col1.isEq("test"), itemId))) shouldBe
(s"CASE WHEN ${col1.name} = 'test' THEN ${itemId.name} ELSE ${shieldId.name} END")
s"CASE WHEN ${col1.name} = 'test' THEN ${itemId.name} ELSE ${shieldId.name} END"
}

it should "generate CONDITIONAL multiIf" in {
Expand All @@ -176,13 +186,13 @@ class ClickhouseTokenizerTest extends DslTestSpec {

// test single case
this.tokenizeColumn(multiIf(shieldId, columnCase(col1.isEq("test"), itemId))) shouldBe
(s"if(${col1.name} = 'test', ${itemId.name}, ${shieldId.name})")
s"if(${col1.name} = 'test', ${itemId.name}, ${shieldId.name})"

// test multi cases
this.tokenizeColumn(
multiIf(shieldId, columnCase(col1.isEq("test"), itemId), columnCase(col1.isEq("test"), itemId))
) shouldBe
(s"multiIf(${col1.name} = 'test', ${itemId.name}, ${col1.name} = 'test', ${itemId.name}, ${shieldId.name})")
s"multiIf(${col1.name} = 'test', ${itemId.name}, ${col1.name} = 'test', ${itemId.name}, ${shieldId.name})"
}

it should "use constant" in {
Expand All @@ -193,43 +203,52 @@ class ClickhouseTokenizerTest extends DslTestSpec {
val col = RawColumn("Robert'); DROP TABLE students;")
val select = SelectQuery(Seq(col))
val query = toSql(InternalQuery(select = Some(select), where = Some(col)))
query should be(s"SELECT ${col.rawSql} WHERE ${col.rawSql} FORMAT JSON")
query should matchSQL(s"SELECT ${col.rawSql} WHERE ${col.rawSql} FORMAT JSON")
}

it should "build with combinators" in {
this.tokenizeColumn(CombinedAggregatedFunction(Combinator.If(col1.isEq("test")), uniq(col1))) shouldBe s"uniqIf(${col1.name},${col1.name} = 'test')"
this.tokenizeColumn(CombinedAggregatedFunction(Combinator.If(col1.isEq("test")), uniqHLL12(col1))) shouldBe s"uniqHLL12If(${col1.name},${col1.name} = 'test')"
this.tokenizeColumn(CombinedAggregatedFunction(Combinator.If(col1.isEq("test")), uniqCombined(col1))) shouldBe s"uniqCombinedIf(${col1.name},${col1.name} = 'test')"
this.tokenizeColumn(CombinedAggregatedFunction(Combinator.If(col1.isEq("test")), uniq(col1))) should matchSQL(
s"uniqIf(${col1.name}, ${col1.name} = 'test')"
)
this.tokenizeColumn(CombinedAggregatedFunction(Combinator.If(col1.isEq("test")), uniqHLL12(col1))) should matchSQL(
s"uniqHLL12If(${col1.name}, ${col1.name} = 'test')"
)
this.tokenizeColumn(
CombinedAggregatedFunction(Combinator.If(col1.isEq("test")), uniqCombined(col1))
) should matchSQL(s"uniqCombinedIf(${col1.name}, ${col1.name} = 'test')")
this.tokenizeColumn(
CombinedAggregatedFunction(Combinator.If(col1.isEq("test")),
CombinedAggregatedFunction(Combinator.If(col2.isEq(3)), uniqExact(col1)))
) shouldBe s"uniqExactIfIf(${col1.name},${col2.name} = 3,${col1.name} = 'test')"
CombinedAggregatedFunction(
Combinator.If(col1.isEq("test")),
CombinedAggregatedFunction(Combinator.If(col2.isEq(3)), uniqExact(col1))
)
) should matchSQL(s"uniqExactIfIf(${col1.name}, ${col2.name} = 3, ${col1.name} = 'test')")
}

it should "uniq for multiple columns" in {
this.tokenizeColumn(uniq(col1, col2)) shouldBe s"uniq(${col1.name},${col2.name})"
this.tokenizeColumn(uniqHLL12(col1, col2)) shouldBe s"uniqHLL12(${col1.name},${col2.name})"
this.tokenizeColumn(uniqExact(col1, col2)) shouldBe s"uniqExact(${col1.name},${col2.name})"
this.tokenizeColumn(uniqCombined(col1, col2)) shouldBe s"uniqCombined(${col1.name},${col2.name})"

this.tokenizeColumn(uniq(col1, col2)) should matchSQL(s"uniq(${col1.name}, ${col2.name})")
this.tokenizeColumn(uniqHLL12(col1, col2)) should matchSQL(s"uniqHLL12(${col1.name}, ${col2.name})")
this.tokenizeColumn(uniqExact(col1, col2)) should matchSQL(s"uniqExact(${col1.name}, ${col2.name})")
this.tokenizeColumn(uniqCombined(col1, col2)) should matchSQL(s"uniqCombined(${col1.name}, ${col2.name})")
}

it should "use zone name for monthly" in {
this.tokenizeTimeSeries(
TimeSeries(
timestampColumn,
MultiInterval(DateTime.now(DateTimeZone.forOffsetHours(2)),
DateTime.now(DateTimeZone.forOffsetHours(2)),
MultiDuration(TimeUnit.Month))
MultiInterval(
DateTime.now(DateTimeZone.forOffsetHours(2)),
DateTime.now(DateTimeZone.forOffsetHours(2)),
MultiDuration(TimeUnit.Month)
)
)
) shouldBe "toDateTime(toStartOfMonth(toDateTime(ts / 1000), 'Etc/GMT-2'), 'Etc/GMT-2')"
) should matchSQL("toDateTime(toStartOfMonth(toDateTime(ts / 1000), 'Etc/GMT-2'), 'Etc/GMT-2')")
}

it should "quote them correctly" in {
val name = "props.key"
val col = RefColumn(name)
val select = SelectQuery(Seq(col))
val query = toSql(InternalQuery(select = Some(select)))
query should be(s"SELECT `$name` FORMAT JSON")
query should matchSQL(s"SELECT `$name` FORMAT JSON")
}
}

0 comments on commit ceaf17f

Please sign in to comment.