From b1b8373399108128adc3b7431ee884620c5558d8 Mon Sep 17 00:00:00 2001 From: xiejiann Date: Fri, 19 Apr 2024 14:13:33 +0800 Subject: [PATCH] add dag for func deps --- .../doris/nereids/properties/FuncDeps.java | 63 ++++++++++++ .../doris/nereids/properties/FuncDepsDAG.java | 99 +++++++++++++++++++ .../nereids/properties/FuncDepsDAGTest.java | 75 ++++++++++++++ 3 files changed, 237 insertions(+) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDepsDAG.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FuncDepsDAGTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java new file mode 100644 index 00000000000000..5138be075b694f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.properties; + +import org.apache.doris.nereids.trees.expressions.Slot; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * Function dependence items. + */ +public class FuncDeps { + class FuncDepsItem { + final Set determinants; + final Set dependencies; + + public FuncDepsItem(Set determinants, Set dependencies) { + this.determinants = determinants; + this.dependencies = dependencies; + } + + @Override + public String toString() { + return String.format("%s -> %s", determinants, dependencies); + } + } + + private final List items; + + FuncDeps() { + items = new ArrayList<>(); + } + + public void addFuncItems(Set determinants, Set dependencies) { + items.add(new FuncDepsItem(determinants, dependencies)); + } + + public int size() { + return items.size(); + } + + @Override + public String toString() { + return items.toString(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDepsDAG.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDepsDAG.java new file mode 100644 index 00000000000000..8431f5a7c7544a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDepsDAG.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.properties; + +import org.apache.doris.nereids.trees.expressions.Slot; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +/** + * Function dependence items. + */ +public class FuncDepsDAG { + class DAGItem { + Set slots; + Set parents; + Set children; + + public DAGItem(Set slots) { + this.slots = slots; + this.parents = new HashSet<>(); + this.children = new HashSet<>(); + } + } + + private Map, DAGItem> itemMap; + + public FuncDepsDAG() { + itemMap = new HashMap<>(); + } + + public void addDeps(Set l, Set r) { + DAGItem lNode = getOrCreateNode(l); + DAGItem rNode = getOrCreateNode(r); + addEdge(lNode, rNode); + } + + private DAGItem getOrCreateNode(Set slots) { + return itemMap.computeIfAbsent(slots, DAGItem::new); + } + + private void addEdge(DAGItem from, DAGItem to) { + if (!from.children.contains(to)) { + from.children.add(to); + to.parents.add(from); + } + } + + /** + * find all func deps + */ + public FuncDeps findValidFuncDeps(Set validSlot) { + FuncDeps res = new FuncDeps(); + for (Entry, DAGItem> entry : itemMap.entrySet()) { + if (validSlot.containsAll(entry.getKey())) { + Set visited = new HashSet<>(); + Set children = new HashSet<>(); + visited.add(entry.getValue()); + collectAllChildren(validSlot, entry.getValue(), visited, children); + for (DAGItem child : children) { + res.addFuncItems(entry.getValue().slots, child.slots); + } + } + } + return res; + } + + private void collectAllChildren(Set validSlot, DAGItem root, + Set visited, Set children) { + for (DAGItem child : root.children) { + if (visited.contains(child)) { + continue; + } + if (validSlot.containsAll(child.slots)) { + children.add(child); + } + visited.add(child); + collectAllChildren(validSlot, child, visited, children); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FuncDepsDAGTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FuncDepsDAGTest.java new file mode 100644 index 00000000000000..eaedbb6232e983 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FuncDepsDAGTest.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.properties; + +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.types.IntegerType; + +import com.google.common.collect.Sets; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class FuncDepsDAGTest { + @Test + void testBasic() { + FuncDepsDAG dag = new FuncDepsDAG(); + Slot s1 = new SlotReference("s1", IntegerType.INSTANCE); + Slot s2 = new SlotReference("s2", IntegerType.INSTANCE); + dag.addDeps(Sets.newHashSet(s1), Sets.newHashSet(s2)); + FuncDeps res = dag.findValidFuncDeps(Sets.newHashSet(s1, s2)); + Assertions.assertEquals(1, res.size()); + } + + @Test + void testTrans() { + FuncDepsDAG dag = new FuncDepsDAG(); + Slot s1 = new SlotReference("s1", IntegerType.INSTANCE); + Slot s2 = new SlotReference("s2", IntegerType.INSTANCE); + Slot s3 = new SlotReference("s3", IntegerType.INSTANCE); + dag.addDeps(Sets.newHashSet(s1), Sets.newHashSet(s2)); + dag.addDeps(Sets.newHashSet(s2), Sets.newHashSet(s3)); + FuncDeps res = dag.findValidFuncDeps(Sets.newHashSet(s1, s3)); + Assertions.assertEquals(1, res.size()); + } + + @Test + void testCircle() { + FuncDepsDAG dag = new FuncDepsDAG(); + Slot s1 = new SlotReference("s1", IntegerType.INSTANCE); + Slot s2 = new SlotReference("s2", IntegerType.INSTANCE); + dag.addDeps(Sets.newHashSet(s1), Sets.newHashSet(s2)); + dag.addDeps(Sets.newHashSet(s2), Sets.newHashSet(s1)); + FuncDeps res = dag.findValidFuncDeps(Sets.newHashSet(s1, s2)); + Assertions.assertEquals(2, res.size()); + } + + @Test + void testTree() { + FuncDepsDAG dag = new FuncDepsDAG(); + Slot s1 = new SlotReference("s1", IntegerType.INSTANCE); + Slot s2 = new SlotReference("s2", IntegerType.INSTANCE); + Slot s3 = new SlotReference("s3", IntegerType.INSTANCE); + Slot s4 = new SlotReference("s4", IntegerType.INSTANCE); + dag.addDeps(Sets.newHashSet(s1), Sets.newHashSet(s2)); + dag.addDeps(Sets.newHashSet(s2), Sets.newHashSet(s3)); + dag.addDeps(Sets.newHashSet(s2), Sets.newHashSet(s4)); + FuncDeps res = dag.findValidFuncDeps(Sets.newHashSet(s1, s4, s3)); + Assertions.assertEquals(2, res.size()); + } +}