[function](bitmap) support bitmap_remove (apache#24190)

hello-stephen · Sep 12, 2023 · 4bb9a12 · 4bb9a12
1 parent 9e0d843
commit 4bb9a12
Show file tree

Hide file tree

Showing 10 changed files with 468 additions and 0 deletions.
diff --git a/be/src/vec/functions/function_bitmap.cpp b/be/src/vec/functions/function_bitmap.cpp
@@ -838,6 +838,42 @@ struct BitmapContains {
     }
 };
 
+struct NameBitmapRemove {
+    static constexpr auto name = "bitmap_remove";
+};
+
+template <typename LeftDataType, typename RightDataType>
+struct BitmapRemove {
+    using ResultDataType = DataTypeBitMap;
+    using T0 = typename LeftDataType::FieldType;
+    using T1 = typename RightDataType::FieldType;
+    using LTData = std::vector<BitmapValue>;
+    using RTData = typename ColumnVector<T1>::Container;
+    using ResTData = std::vector<BitmapValue>;
+
+    static void vector_vector(const LTData& lvec, const RTData& rvec, ResTData& res) {
+        size_t size = lvec.size();
+        for (size_t i = 0; i < size; ++i) {
+            res[i] = lvec[i];
+            res[i].remove(rvec[i]);
+        }
+    }
+    static void vector_scalar(const LTData& lvec, const T1& rval, ResTData& res) {
+        size_t size = lvec.size();
+        for (size_t i = 0; i < size; ++i) {
+            res[i] = lvec[i];
+            res[i].remove(rval);
+        }
+    }
+    static void scalar_vector(const BitmapValue& lval, const RTData& rvec, ResTData& res) {
+        size_t size = rvec.size();
+        for (size_t i = 0; i < size; ++i) {
+            res[i] = lval;
+            res[i].remove(rvec[i]);
+        }
+    }
+};
+
 struct NameBitmapHasAny {
     static constexpr auto name = "bitmap_has_any";
 };
@@ -1227,6 +1263,8 @@ using FunctionBitmapAndNot =
         FunctionBinaryToType<DataTypeBitMap, DataTypeBitMap, BitmapAndNot, NameBitmapAndNot>;
 using FunctionBitmapContains =
         FunctionBinaryToType<DataTypeBitMap, DataTypeInt64, BitmapContains, NameBitmapContains>;
+using FunctionBitmapRemove =
+        FunctionBinaryToType<DataTypeBitMap, DataTypeInt64, BitmapRemove, NameBitmapRemove>;
 
 using FunctionBitmapHasAny =
         FunctionBinaryToType<DataTypeBitMap, DataTypeBitMap, BitmapHasAny, NameBitmapHasAny>;
@@ -1254,6 +1292,7 @@ void register_function_bitmap(SimpleFunctionFactory& factory) {
     factory.register_function<FunctionBitmapAndNot>();
     factory.register_function<FunctionBitmapAndNotCount>();
     factory.register_function<FunctionBitmapContains>();
+    factory.register_function<FunctionBitmapRemove>();
     factory.register_function<FunctionBitmapHasAny>();
     factory.register_function<FunctionBitmapHasAll>();
     factory.register_function<FunctionSubBitmap>();

diff --git a/be/test/vec/function/function_bitmap_test.cpp b/be/test/vec/function/function_bitmap_test.cpp
@@ -82,6 +82,23 @@ TEST(function_bitmap_test, function_bitmap_to_string_test) {
     check_function<DataTypeString, true>(func_name, input_types, data_set);
 }
 
+TEST(function_bitmap_test, function_bitmap_remove) {
+    std::string func_name = "bitmap_remove";
+    InputTypeSet input_types = {TypeIndex::BitMap, TypeIndex::Int64};
+
+    BitmapValue bitmap1({1, 3});
+    BitmapValue bitmap2({1, 3, 5});
+
+    BitmapValue bitmap1_res(1);
+    BitmapValue bitmap2_res({1, 3, 5});
+    {
+        DataSet data_set = {{{&bitmap1, (int64_t)3}, bitmap1_res},
+                            {{&bitmap2, (int64_t)6}, bitmap2_res},
+                            {{&bitmap1, Null()}, Null()}};
+
+        check_function<DataTypeBitMap, true>(func_name, input_types, data_set);
+    }
+}
 namespace doris {
 namespace config {
 DECLARE_Bool(enable_set_in_bitmap_value);

diff --git a/docs/en/docs/sql-manual/sql-functions/bitmap-functions/bitmap-remove.md b/docs/en/docs/sql-manual/sql-functions/bitmap-functions/bitmap-remove.md
@@ -0,0 +1,55 @@
+---
+{
+    "title": "BITMAP_REMOVE",
+    "language": "en"
+}
+---
+
+<!-- 
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## bitmap_remove
+### description
+#### Syntax
+
+`BITMAP BITMAP_REMOVE(BITMAP bitmap, BIGINT input)`
+
+Remove the specified value from bitmap.
+
+### example
+
+```
+mysql [(none)]>select bitmap_to_string(bitmap_remove(bitmap_from_string('1, 2, 3'), 3)) res; 
++------+
+| res  |
++------+
+| 1,2  |
++------+
+
+mysql [(none)]>select bitmap_to_string(bitmap_remove(bitmap_from_string('1, 2, 3'), null)) res;
++------+
+| res  |
++------+
+| NULL |
++------+
+```
+
+### keywords
+
+    BITMAP_REMOVE,BITMAP
diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/bitmap-functions/bitmap-remove.md b/docs/zh-CN/docs/sql-manual/sql-functions/bitmap-functions/bitmap-remove.md
@@ -0,0 +1,55 @@
+---
+{
+    "title": "BITMAP_REMOVE",
+    "language": "zh-CN"
+}
+---
+
+<!-- 
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## bitmap_remove
+### description
+#### Syntax
+
+`BITMAP BITMAP_REMOVE(BITMAP bitmap, BIGINT input)`
+
+从Bitmap列中删除指定的值。
+
+### example
+
+```
+mysql [(none)]>select bitmap_to_string(bitmap_remove(bitmap_from_string('1, 2, 3'), 3)) res; 
++------+
+| res  |
++------+
+| 1,2  |
++------+
+
+mysql [(none)]>select bitmap_to_string(bitmap_remove(bitmap_from_string('1, 2, 3'), null)) res;
++------+
+| res  |
++------+
+| NULL |
++------+
+```
+
+### keywords
+
+    BITMAP_REMOVE,BITMAP
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
@@ -74,6 +74,7 @@
 import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapNot;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapOr;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapOrCount;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapRemove;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapSubsetInRange;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapSubsetLimit;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapToArray;
@@ -430,6 +431,7 @@ public class BuiltinScalarFunctions implements FunctionHelper {
             scalar(BitmapNot.class, "bitmap_not"),
             scalar(BitmapOr.class, "bitmap_or"),
             scalar(BitmapOrCount.class, "bitmap_or_count"),
+            scalar(BitmapRemove.class, "bitmap_remove"),
             scalar(BitmapSubsetInRange.class, "bitmap_subset_in_range"),
             scalar(BitmapSubsetLimit.class, "bitmap_subset_limit"),
             scalar(BitmapToArray.class, "bitmap_to_array"),

diff --git a/...c/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapRemove.java b/...c/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapRemove.java
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.scalar;
+
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
+import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.BigIntType;
+import org.apache.doris.nereids.types.BitmapType;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * ScalarFunction 'bitmap_remove'. This class is generated by GenerateFunction.
+ */
+public class BitmapRemove extends ScalarFunction
+        implements BinaryExpression, ExplicitlyCastableSignature, PropagateNullable {
+
+    public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+            FunctionSignature.ret(BitmapType.INSTANCE).args(BitmapType.INSTANCE, BigIntType.INSTANCE)
+    );
+
+    /**
+     * constructor with 2 arguments.
+     */
+    public BitmapRemove(Expression arg0, Expression arg1) {
+        super("bitmap_remove", arg0, arg1);
+    }
+
+    /**
+     * withChildren.
+     */
+    @Override
+    public BitmapRemove withChildren(List<Expression> children) {
+        Preconditions.checkArgument(children.size() == 2, String.format("children.size() is %d", children.size()));
+        return new BitmapRemove(children.get(0), children.get(1));
+    }
+
+    @Override
+    public List<FunctionSignature> getSignatures() {
+        return SIGNATURES;
+    }
+
+    @Override
+    public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+        return visitor.visitBitmapRemove(this, context);
+    }
+}
diff --git a/...c/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/...c/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
@@ -78,6 +78,7 @@
 import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapNot;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapOr;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapOrCount;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapRemove;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapSubsetInRange;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapSubsetLimit;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapToArray;
@@ -596,6 +597,10 @@ default R visitBitmapOrCount(BitmapOrCount bitmapOrCount, C context) {
         return visitScalarFunction(bitmapOrCount, context);
     }
 
+    default R visitBitmapRemove(BitmapRemove bitmapRemove, C context) {
+        return visitScalarFunction(bitmapRemove, context);
+    }
+
     default R visitBitmapSubsetInRange(BitmapSubsetInRange bitmapSubsetInRange, C context) {
         return visitScalarFunction(bitmapSubsetInRange, context);
     }

diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py
@@ -1836,6 +1836,7 @@
         [['bitmap_from_array'], 'BITMAP', ['ARRAY_INT'], 'ALWAYS_NULLABLE'],
         [['bitmap_from_array'], 'BITMAP', ['ARRAY_BIGINT'], 'ALWAYS_NULLABLE'],
         [['bitmap_contains'], 'BOOLEAN', ['BITMAP','BIGINT'], ''],
+        [['bitmap_remove'], 'BITMAP', ['BITMAP','BIGINT'], ''],
         [['bitmap_has_any'], 'BOOLEAN', ['BITMAP','BITMAP'], ''],
         [['bitmap_has_all'], 'BOOLEAN', ['BITMAP','BITMAP'], ''],
         [['bitmap_min'], 'BIGINT', ['BITMAP'], 'ALWAYS_NULLABLE'],