branch-3.1: [fix] (function) Support concat_ws multi array parameters apache#53084 apache#53835 (apache#54196)

picked from apache#53084 apache#53835

Author: dwdwqfwe

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java

@@ -18,12 +18,14 @@
 package org.apache.doris.nereids.rules.expression;
 
 import org.apache.doris.nereids.rules.expression.check.CheckCast;
+import org.apache.doris.nereids.rules.expression.rules.ConcatWsMultiArrayToOne;
 import org.apache.doris.nereids.rules.expression.rules.ConvertAggStateCast;
 import org.apache.doris.nereids.rules.expression.rules.DigitalMaskingConvert;
 import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule;
 import org.apache.doris.nereids.rules.expression.rules.InPredicateDedup;
 import org.apache.doris.nereids.rules.expression.rules.InPredicateExtractNonConstant;
 import org.apache.doris.nereids.rules.expression.rules.InPredicateToEqualToRule;
+import org.apache.doris.nereids.rules.expression.rules.LogToLn;
 import org.apache.doris.nereids.rules.expression.rules.MergeDateTrunc;
 import org.apache.doris.nereids.rules.expression.rules.NormalizeBinaryPredicatesRule;
 import org.apache.doris.nereids.rules.expression.rules.SimplifyArithmeticComparisonRule;
@@ -51,6 +53,8 @@ public class ExpressionNormalization extends ExpressionRewrite {
                 InPredicateToEqualToRule.INSTANCE,
                 SimplifyNotExprRule.INSTANCE,
                 SimplifyArithmeticRule.INSTANCE,
+                LogToLn.INSTANCE,
+                ConcatWsMultiArrayToOne.INSTANCE,
                 FoldConstantRule.INSTANCE,
                 SimplifyCastRule.INSTANCE,
                 DigitalMaskingConvert.INSTANCE,

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java

@@ -24,7 +24,6 @@
 import org.apache.doris.nereids.rules.expression.rules.DistinctPredicatesRule;
 import org.apache.doris.nereids.rules.expression.rules.ExtractCommonFactorRule;
 import org.apache.doris.nereids.rules.expression.rules.LikeToEqualRewrite;
-import org.apache.doris.nereids.rules.expression.rules.LogToLn;
 import org.apache.doris.nereids.rules.expression.rules.NullSafeEqualToEqual;
 import org.apache.doris.nereids.rules.expression.rules.SimplifyComparisonPredicate;
 import org.apache.doris.nereids.rules.expression.rules.SimplifyConflictCompound;
@@ -62,8 +61,7 @@ public class ExpressionOptimization extends ExpressionRewrite {
                 TopnToMax.INSTANCE,
                 NullSafeEqualToEqual.INSTANCE,
                 LikeToEqualRewrite.INSTANCE,
-                BetweenToEqual.INSTANCE,
-                LogToLn.INSTANCE
+                BetweenToEqual.INSTANCE
             )
     );
     private static final ExpressionRuleExecutor EXECUTOR = new ExpressionRuleExecutor(OPTIMIZE_REWRITE_RULES);

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ConcatWsMultiArrayToOne.java

@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression.rules;
+
+import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher;
+import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayConcat;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ConcatWs;
+import org.apache.doris.nereids.types.ArrayType;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+    * ConcatWsTMultiArrayToOne, convert ConcatWs with multiple array arguments to a single array argument.
+    * This rule is useful for optimizing queries that use ConcatWs with multiple array arguments,
+    * allowing them to be processed more efficiently by combining the arrays into a single array argument.
+ */
+public class ConcatWsMultiArrayToOne implements ExpressionPatternRuleFactory {
+
+    public static final ConcatWsMultiArrayToOne INSTANCE = new ConcatWsMultiArrayToOne();
+
+    @Override
+    public List<ExpressionPatternMatcher<? extends Expression>> buildRules() {
+        return ImmutableList.of(
+                matchesType(ConcatWs.class).then(ConcatWsMultiArrayToOne::rewrite)
+        );
+    }
+
+    /** rewrite */
+    public static Expression rewrite(ConcatWs cat) {
+        if (cat.arity() >= 3 && cat.child(2).getDataType() instanceof ArrayType) {
+            return new ConcatWs(cat.child(0),
+                    new ArrayConcat(cat.child(1), cat.children().subList(2, cat.arity()).toArray(new Expression[0])));
+        } else {
+            return cat;
+        }
+    }
+}

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConcatWs.java

@@ -42,8 +42,12 @@ public class ConcatWs extends ScalarFunction
                     .args(VarcharType.SYSTEM_DEFAULT, ArrayType.of(VarcharType.SYSTEM_DEFAULT)),
             FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT)
                     .varArgs(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT),
-            FunctionSignature.ret(StringType.INSTANCE).varArgs(StringType.INSTANCE, StringType.INSTANCE)
-    );
+            FunctionSignature.ret(StringType.INSTANCE).varArgs(StringType.INSTANCE, StringType.INSTANCE),
+            // This signature is for compatibility with the old implementation
+            FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT)
+                    .varArgs(VarcharType.SYSTEM_DEFAULT, ArrayType.of(VarcharType.SYSTEM_DEFAULT)),
+            FunctionSignature.ret(StringType.INSTANCE)
+                    .varArgs(StringType.INSTANCE, ArrayType.of(StringType.INSTANCE)));
 
     /**
      * constructor with 2 or more arguments.

regression-test/data/nereids_function_p0/scalar_function/concat_ws.out

@@ -0,0 +1,50 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !concat_ws_ArrayWithNullElement --
+a-b-css-d-g-f-s
+
+-- !concat_ws_ArrayWithEmptyString --
+a---css-d-
+
+-- !concat_ws_WithEmptyArray --
+a-b-css-d
+
+-- !concat_ws_SeparatorSpecial --
+x|y|m|n|p
+
+-- !concat_ws_SeparatorEmpty --
+abcd
+
+-- !concat_ws_ArrayWithNumber --
+1-2-3-4-5-6
+
+-- !concat_ws_WithNullArray --
+
+
+-- !concat_ws_SingleArray --
+x,y,z
+
+-- !concat_ws_ArrayAllNull --
+a-b
+
+-- !concat_ws_MixedTypeElement --
+a|123|456|b
+
+-- !concat_ws_chinese --
+你好，世界，Doris，Nereids，测试
+
+-- !concat_ws_insert_1 --
+a-b-css-d
+x-y-z
+你好-世界-Doris-Nereids
+
+-- !log_function --
+2.0	3.0	3.0	3.0000000000000004	2.9999999999999996
+
+-- !log_function_with_null --
+\N	\N	\N	\N	\N
+
+-- !log_function_wiht_one_argument --
+4.605170185988092	2.0794415416798357	3.295836866004329	4.8283137373023015	6.907755278982137
+
+-- !log_function_with_null_and_one_argument --
+\N	\N	0.0	2.302585092994046

regression-test/suites/nereids_function_p0/scalar_function/concat_ws.groovy

@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("nereids_scalar_fn_concat_ws_and_log") {
+
+    sql 'set enable_fold_constant_by_be=true'
+    qt_concat_ws_ArrayWithNullElement "select concat_ws('-',['a','b'],['css',null,'d'],['g','f'],['s'])"
+
+
+    qt_concat_ws_ArrayWithEmptyString "select concat_ws('-',['a',''],['','css'],['d',''])"
+    qt_concat_ws_WithEmptyArray "select concat_ws('-',['a','b'],[],['css','d'],[])"
+    qt_concat_ws_SeparatorSpecial "select concat_ws('|',['x','y'],['m',null,'n'],['p'])"
+
+    qt_concat_ws_SeparatorEmpty "select concat_ws('',['a','b'],['c',null],['d'])"
+
+    qt_concat_ws_ArrayWithNumber "select concat_ws('-',['1','2'],['3',null,'4'],['5','6'])"
+    qt_concat_ws_WithNullArray "select concat_ws('-',['a'],null,['b','c'])"
+    qt_concat_ws_SingleArray "select concat_ws(',',['x','y','z'])"
+    qt_concat_ws_ArrayAllNull "select concat_ws('-',['a'],[null,null],['b'])"
+    qt_concat_ws_MixedTypeElement "select concat_ws('|',['a','123'],['456',null,'b'])"
+    qt_concat_ws_chinese "select concat_ws('，',['你好','世界'],['Doris',null,'Nereids'],['测试'])"
+
+    sql "DROP TABLE IF EXISTS test_concat_ws_1"
+    sql "CREATE TABLE test_concat_ws_1 (id INT, a ARRAY<VARCHAR>, b ARRAY<VARCHAR>) ENGINE=OLAP DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES ('replication_num' = '1')"
+    sql "INSERT INTO test_concat_ws_1 VALUES (1, ['a','b'], ['css',null,'d']), (2, ['x',null], ['y','z']),(3,['你好','世界'],['Doris',null,'Nereids'])"
+    qt_concat_ws_insert_1 "SELECT concat_ws('-', a, b) FROM test_concat_ws_1 ORDER BY id"
+
+
+    qt_log_function "SELECT log(10, 100), log(2, 8), log(3, 27), log(5, 125), log(10, 1000)"
+    qt_log_function_with_null "SELECT log(10, NULL), log(NULL, 100), log(NULL, NULL), log(10, 0), log(0, 10)"
+    qt_log_function_wiht_one_argument "SELECT log(100), log(8), log(27), log(125), log(1000)"
+    qt_log_function_with_null_and_one_argument "SELECT log(NULL), log(0), log(1), log(10)"
+}