Skip to content

Commit

Permalink
[CALCITE-6599] RelMdPredicates pull up wrong predicate from VALUES
Browse files Browse the repository at this point in the history
  • Loading branch information
@NobiGo
NobiGo committed Sep 29, 2024
1 parent a4a27e3 commit 36e0727
Show file tree
Hide file tree
Showing 3 changed files with 111 additions and 30 deletions.
60 changes: 40 additions & 20 deletions core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.core.Union;
import org.apache.calcite.rel.core.Values;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexExecutor;
Expand All @@ -54,16 +55,21 @@
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.fun.SqlInternalOperators;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.BitSets;
import org.apache.calcite.util.Bug;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Sarg;
import org.apache.calcite.util.Util;
import org.apache.calcite.util.mapping.Mapping;
import org.apache.calcite.util.mapping.MappingType;
import org.apache.calcite.util.mapping.Mappings;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Range;
import com.google.common.collect.RangeSet;
import com.google.common.collect.TreeRangeSet;

import org.checkerframework.checker.nullness.qual.Nullable;

Expand All @@ -77,7 +83,6 @@
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
Expand Down Expand Up @@ -562,34 +567,49 @@ public RelOptPredicateList getPredicates(Exchange exchange,
/**
* Infers predicates for a Values.
*
* <p>The predicates on {@code T (x, y, z)} with rows
* {@code (1, 2, null), (1, 2, null), (5, 2, null)} are {@code 'y = 2'} and {@code 'z is null'}.
* <p>The predicates on {@code T (w, x, y, z)} with rows
* {@code (1, 2, 3, null), (1, 2, null, null), (5, 2, 3, null)} are
* {@code 'SEARCH($0, Sarg[1, 5])'},
* {@code 'SEARCH($1, Sarg[2])'},
* {@code 'SEARCH($2, Sarg[3; NULL AS TRUE])'} and
* {@code '[IS NULL($3)'}.
*/
public RelOptPredicateList getPredicates(Values values, RelMetadataQuery mq) {
ImmutableList<ImmutableList<RexLiteral>> tuples = values.tuples;
if (!tuples.isEmpty()) {
Set<Integer> constants = new HashSet<>();
IntStream.range(0, tuples.size()).boxed().forEach(constants::add);
List<RexLiteral> firstTuple = new ArrayList<>(tuples.get(0));
for (ImmutableList<RexLiteral> tuple : tuples) {
if (constants.isEmpty()) {
break;
}
for (int i = 0; i < tuple.size(); i++) {
if (!Objects.equals(tuple.get(i), firstTuple.get(i))) {
constants.remove(i);
List<RexLiteral> firstTuple = tuples.get(0);
List<RangeSet<Comparable>> valueRangeSet = new ArrayList<>();
List<RexUnknownAs> unKnownValueLIst = new ArrayList<>();
IntStream.range(0, firstTuple.size()).boxed().forEach(value-> {

Check failure on line 583 in core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java

View workflow job for this annotation

GitHub Actions / Linux (JDK 21) 

[Task :core:checkstyleMain] [WhitespaceAround] '->' is not preceded with whitespace.

Check failure on line 583 in core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java

View workflow job for this annotation

GitHub Actions / Linux (JDK 17) 

[Task :core:checkstyleMain] [WhitespaceAround] '->' is not preceded with whitespace.

Check failure on line 583 in core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java

View workflow job for this annotation

GitHub Actions / Linux (JDK 8, oldest Guava, America/New_York Timezone) 

[Task :core:checkstyleMain] [WhitespaceAround] '->' is not preceded with whitespace.

Check failure on line 583 in core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java

View workflow job for this annotation

GitHub Actions / Linux (JDK 11, Pacific/Chatham Timezone) 

[Task :core:checkstyleMain] [WhitespaceAround] '->' is not preceded with whitespace.

Check failure on line 583 in core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java

View workflow job for this annotation

GitHub Actions / Linux (JDK 8, latest Guava, America/New_York Timezone) 

[Task :core:checkstyleMain] [WhitespaceAround] '->' is not preceded with whitespace.

Check failure on line 583 in core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java

View workflow job for this annotation

GitHub Actions / Linux (JDK 23) 

[Task :core:checkstyleMain] [WhitespaceAround] '->' is not preceded with whitespace.

Check failure on line 583 in core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java

View workflow job for this annotation

GitHub Actions / Linux (JDK 11, Avatica main) 

[Task :core:checkstyleMain] [WhitespaceAround] '->' is not preceded with whitespace.

Check failure on line 583 in core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java

View workflow job for this annotation

GitHub Actions / macOS (JDK 21) 

[Task :core:checkstyleMain] [WhitespaceAround] '->' is not preceded with whitespace.
valueRangeSet.add(value, TreeRangeSet.create());
unKnownValueLIst.add(RexUnknownAs.UNKNOWN);
});
for (int i = 0; i < tuples.size(); i++) {
List<RexLiteral> tuple = tuples.get(i);
for (int j = 0; j < tuple.size(); j++) {
RexLiteral rexLiteral = tuple.get(j);
if (RexLiteral.isNullLiteral(rexLiteral)) {
unKnownValueLIst.set(j, RexUnknownAs.TRUE);
} else {
valueRangeSet.get(j).add(Range.singleton(requireNonNull(rexLiteral.getValue5())));
}
}
}
RexBuilder rexBuilder = values.getCluster().getRexBuilder();
List<RexNode> predicates = new ArrayList<>();
for (int i = 0; i < firstTuple.size(); i++) {
if (constants.contains(i)) {
RexLiteral literal = firstTuple.get(i);
final Sarg sarg = Sarg.of(unKnownValueLIst.get(i), valueRangeSet.get(i));
RelDataType relDataType = values.getRowType().getFieldList().get(i).getType();
if (sarg.isNone() && relDataType.getSqlTypeName() == SqlTypeName.NULL) {
predicates.add(
rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
rexBuilder.makeInputRef(literal.getType(), i),
literal));
eqConstant(
values, rexBuilder, i, rexBuilder.makeNullLiteral(
rexBuilder.getTypeFactory().createSqlType(SqlTypeName.NULL))));
} else {
predicates.add(
i, rexBuilder.makeCall(SqlStdOperatorTable.SEARCH,
rexBuilder.makeInputRef(values, i),
rexBuilder.makeSearchArgumentLiteral(sarg, relDataType)));
}
}
return RelOptPredicateList.of(rexBuilder, predicates);
Expand Down Expand Up @@ -815,7 +835,7 @@ public RelOptPredicateList inferPredicates(
RelOptUtil.conjunctions(joinRel.getCondition())),
inferredPredicates);
return RelOptPredicateList.of(rexBuilder, pulledUpPredicates,
leftInferredPredicates, rightInferredPredicates);
leftInferredPredicates, rightInferredPredicates);
case LEFT:
case ANTI:
return RelOptPredicateList.of(rexBuilder,
Expand Down Expand Up @@ -851,7 +871,7 @@ private void infer(@Nullable RexNode predicates, Set<RexNode> allExprs,
RexNode tr =
r.accept(
new RexPermuteInputsShuttle(m, joinRel.getInput(0),
joinRel.getInput(1)));
joinRel.getInput(1)));
// Filter predicates can be already simplified, so we should work with
// simplified RexNode versions as well. It also allows prevent of having
// some duplicates in in result pulledUpPredicates
Expand Down
18 changes: 18 additions & 0 deletions core/src/main/java/org/apache/calcite/rex/RexLiteral.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1027,6 +1027,24 @@ public boolean isNull() {
}
}

public @Nullable Comparable getValue5() {
if (value == null) {
return null;
}
switch (typeName) {
case TIMESTAMP:
case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
return getValueAs(TimestampString.class);
case DATE:
return getValueAs(DateString.class);
case TIME:
case TIME_WITH_LOCAL_TIME_ZONE:
return getValueAs(TimeString.class);
default:
return value;
}
}

/** Returns the value of this literal as an instance of the specified class.
*
* <p>The following SQL types allow more than one form:
Expand Down
63 changes: 53 additions & 10 deletions core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2596,6 +2596,49 @@ private void checkPredicates(RelOptCluster cluster, RelOptTable empTable,

}

/** Test case for
* <a href="https://issues.apache.org/jira/browse/CALCITE-6599">[CALCITE-6599]
* RelMdPredicates pull up wrong predicate from VALUES</a>. */
@Test void testPullUpPredicatesFromValues1() {
final String sql = "values(1, 2, 3)";
final Values values = (Values) sql(sql).toRel();
final RelMetadataQuery mq = values.getCluster().getMetadataQuery();
RelOptPredicateList inputSet = mq.getPulledUpPredicates(values);
ImmutableList<RexNode> pulledUpPredicates = inputSet.pulledUpPredicates;
assertThat(pulledUpPredicates,
sortsAs("[SEARCH($0, Sarg[1]), SEARCH($1, Sarg[2]), SEARCH($2, Sarg[3])]"));
}

@Test void testPullUpPredicatesFromValues2() {
final String sql = "values(cast(null as integer), null)";
final Values values = (Values) sql(sql).toRel();
final RelMetadataQuery mq = values.getCluster().getMetadataQuery();
RelOptPredicateList inputSet = mq.getPulledUpPredicates(values);
ImmutableList<RexNode> pulledUpPredicates = inputSet.pulledUpPredicates;
assertThat(pulledUpPredicates, sortsAs("[IS NULL($1), SEARCH($0, Sarg[IS NULL])]"));
}

@Test void testPullUpPredicatesFromValues3() {
final String sql = "values(1, 2, 3, null)";
final Values values = (Values) sql(sql).toRel();
final RelMetadataQuery mq = values.getCluster().getMetadataQuery();
RelOptPredicateList inputSet = mq.getPulledUpPredicates(values);
ImmutableList<RexNode> pulledUpPredicates = inputSet.pulledUpPredicates;
assertThat(pulledUpPredicates,
sortsAs("[IS NULL($3), SEARCH($0, Sarg[1]), SEARCH($1, Sarg[2]), SEARCH($2, Sarg[3])]"));
}

@Test void testPullUpPredicatesFromValues4() {
final String sql = "values(1, 2, 3, null), (1, 2, null, null), (5, 2, 3, null)";
final Values values = (Values) sql(sql).toRel();
final RelMetadataQuery mq = values.getCluster().getMetadataQuery();
RelOptPredicateList inputSet = mq.getPulledUpPredicates(values);
ImmutableList<RexNode> pulledUpPredicates = inputSet.pulledUpPredicates;
assertThat(pulledUpPredicates,
sortsAs("[IS NULL($3), SEARCH($0, Sarg[1, 5]), "
+ "SEARCH($1, Sarg[2]), SEARCH($2, Sarg[3; NULL AS TRUE])]"));
}

@Test void testPullUpPredicatesFromIntersect0() {
final RelNode rel = sql(""
+ "select empno from emp where empno=1\n"
Expand Down Expand Up @@ -3617,8 +3660,8 @@ public void checkAllPredicatesAndTableSetOp(String sql) {
final String sql = "select * from emp order by ename limit 10";
sql(sql)
.assertThatNodeTypeCountIs(TableScan.class, 1,
Project.class, 1,
Sort.class, 1);
Project.class, 1,
Sort.class, 1);
}

@Test void testNodeTypeCountSortLimitOffset() {
Expand Down Expand Up @@ -3782,13 +3825,13 @@ public void checkAllPredicatesAndTableSetOp(String sql) {
.scan("EMP")
.scan("DEPT")
.join(JoinRelType.INNER,
builder.call(SqlStdOperatorTable.EQUALS,
builder.field(2, 0, 0),
builder.field(2, 1, 0)))
builder.call(SqlStdOperatorTable.EQUALS,
builder.field(2, 0, 0),
builder.field(2, 1, 0)))
.build();
assertThat(mq.getPulledUpPredicates(join1)
.pulledUpPredicates
.get(0),
.pulledUpPredicates
.get(0),
hasToString("=($0, $8)"));
}

Expand All @@ -3804,9 +3847,9 @@ public void checkAllPredicatesAndTableSetOp(String sql) {
RelNode filter1 = builder
.scan("EMP")
.filter(
builder.call(SqlStdOperatorTable.EQUALS,
builder.field(1, 0, 0),
builder.field(1, 0, 1)))
builder.call(SqlStdOperatorTable.EQUALS,
builder.field(1, 0, 0),
builder.field(1, 0, 1)))
.build();
assertThat(mq.getPulledUpPredicates(filter1)
.pulledUpPredicates
Expand Down

0 comments on commit 36e0727

Please sign in to comment.