support arabic-indic digits date.

lgbo-ustc · lgbo-ustc · commit 28d7d298df91 · 2025-11-05T12:18:03.000+08:00
diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
@@ -63,6 +63,12 @@ class GlutenFunctionValidateSuite extends GlutenClickHouseWholeStageTransformerS
       .set("spark.sql.shuffle.partitions", "5")
       .set("spark.sql.autoBroadcastJoinThreshold", "10MB")
       .set(GlutenConfig.GLUTEN_SUPPORTED_SCALA_UDFS.key, "compare_substrings:compare_substrings")
+      .set(
+        "spark.sql.optimizer.excludedRules",
+        "org.apache.spark.sql.catalyst.optimizer.ConstantFolding" +
+          "," +
+          "org.apache.spark.sql.catalyst.optimizer.NullPropagation"
+      )
   }
 
   override def beforeAll(): Unit = {
@@ -1386,6 +1392,38 @@ class GlutenFunctionValidateSuite extends GlutenClickHouseWholeStageTransformerS
     }
   }
 
+  test("arabic_indic digit date") {
+    withSQLConf(
+      SQLConf.OPTIMIZER_EXCLUDED_RULES.key ->
+        (ConstantFolding.ruleName + "," + NullPropagation.ruleName)) {
+      sql("create table tb_arabic_date(d string) using parquet")
+      sql("""
+            |insert into tb_arabic_date values
+            |('2020-01-01'),
+            |(cast(unbase64('2aLZoNmi2aQt2aDZpi3ZoNmh') as string)),
+            |(cast(unbase64('2aLZoNmi2aQt2aHZoi3Zo9mh') as string)),
+            |('2022-10-11'),
+            |(cast(unbase64('2aLZoNmi2aQt2aHZoi3Zo9mh') as string))
+            |""".stripMargin)
+      var query_sql = "select from_unixtime(unix_timestamp(d, 'yyyy-MM-dd')) from tb_arabic_date"
+      compareResultsAgainstVanillaSpark(query_sql, true, { _ => })
+
+      query_sql = """
+                    |select from_unixtime(
+                    | unix_timestamp(cast(unbase64('2aLZoNmi2aQt2aDZpi3ZoNmh') as string),
+                    | 'yyyy-MM-dd'))
+                    |""".stripMargin
+      compareResultsAgainstVanillaSpark(query_sql, true, { _ => })
+
+      query_sql = """
+                    |select from_unixtime(unix_timestamp('2020-01-01', 'yyyy-MM-dd'))
+                    |""".stripMargin
+      compareResultsAgainstVanillaSpark(query_sql, true, { _ => })
+
+      sql("drop table tb_arabic_date")
+    }
+  }
+
   test("Test map with nullable key") {
     val sql = "select map(string_field1, int_field1) from json_test where string_field1 is not null"
     compareResultsAgainstVanillaSpark(sql, true, { _ => })
diff --git a/cpp-ch/local-engine/Functions/ArabicIndicToAsciiDigitForDate.cpp b/cpp-ch/local-engine/Functions/ArabicIndicToAsciiDigitForDate.cpp
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#include <Columns/IColumn.h>
+#include <Columns/ColumnNullable.h>
+#include <Columns/ColumnConst.h>
+#include <Columns/ColumnString.h>
+#include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/IDataType.h>
+#include <Functions/FunctionFactory.h>
+#include <Functions/FunctionHelpers.h>
+#include <Functions/IFunction.h>
+#include <Common/Exception.h>
+#include <Common/logger_useful.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+}
+}
+
+namespace local_engine
+{
+// Since spark 3.3, unix_timestamp support arabic number input, e.g., "٢٠٢١-٠٧-٠١ ١٢:٠٠:٠٠".
+// We implement a function to translate arabic indic digits to ascii digits here.
+class ArabicIndicToAsciiDigitForDateFunction : public DB::IFunction
+{
+public:
+    static constexpr auto name = "arabic_indic_to_ascii_digit_for_date";
+
+    static DB::FunctionPtr create(DB::ContextPtr) { return std::make_shared<ArabicIndicToAsciiDigitForDateFunction>(); }
+
+    String getName() const override { return name; }
+
+    bool isSuitableForShortCircuitArgumentsExecution(const DB::DataTypesWithConstInfo & /*arguments*/) const override { return false; }
+    size_t getNumberOfArguments() const override { return 1; }
+
+    DB::DataTypePtr getReturnTypeImpl(const DB::DataTypes & arguments) const override
+    {
+        auto nested_type = DB::removeNullable(arguments[0]);
+        if (!DB::WhichDataType(nested_type).isString())
+            throw DB::Exception(DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Argument for function {} must be String, but got {}", getName(), arguments[0]->getName());
+        return arguments[0];
+    }
+
+    DB::ColumnPtr executeImpl(const DB::ColumnsWithTypeAndName & arguments, const DB::DataTypePtr &, size_t input_rows_count) const override
+    {
+        auto data_col = arguments[0].column;
+        const DB::ColumnString * col_str = nullptr;
+        const DB::ColumnNullable * col_nullable = nullptr;
+        const DB::NullMap * null_map = nullptr;
+        if (data_col->isConst())
+        {
+            if (data_col->isNullAt(0))
+            {
+                return data_col;
+            }
+            const DB::ColumnConst * col_const = DB::checkAndGetColumn<DB::ColumnConst>(data_col.get());
+            data_col = col_const->getDataColumnPtr();
+            if (data_col->isNullable())
+            {
+                col_nullable = DB::checkAndGetColumn<DB::ColumnNullable>(data_col.get());
+                null_map = &(col_nullable->getNullMapData());
+                col_str = DB::checkAndGetColumn<DB::ColumnString>(&(col_nullable->getNestedColumn()));
+            }
+            else
+            {
+                col_str = DB::checkAndGetColumn<DB::ColumnString>(data_col.get());
+            }
+            if (!col_str)
+                throw DB::Exception(DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Argument for function {} must be String, but got {}", getName(), data_col->getName());
+            auto date_str = col_str->getDataAt(0);
+            auto new_str = convertArabicIndicDigit(date_str);
+            auto new_data_col = data_col->cloneEmpty();
+            new_data_col->insertData(new_str.c_str(), new_str.size());
+            return DB::ColumnConst::create(std::move(new_data_col), input_rows_count);
+        }
+
+        if (data_col->isNullable())
+        {
+            col_nullable = DB::checkAndGetColumn<DB::ColumnNullable>(data_col.get());
+            null_map = &(col_nullable->getNullMapData());
+            col_str = DB::checkAndGetColumn<DB::ColumnString>(&(col_nullable->getNestedColumn()));
+        }
+        else
+        {
+            col_str = DB::checkAndGetColumn<DB::ColumnString>(data_col.get());
+        }
+        if (!col_str)
+            throw DB::Exception(DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Argument for function {} must be String, but got {}", getName(), data_col->getName());
+
+        auto nested_data_col = DB::removeNullable(arguments[0].column);
+        bool has_arabic_indic_digit = false;
+        size_t row_index = 0;
+        for (row_index = 0; row_index < input_rows_count; ++row_index)
+        {
+            if (null_map && (*null_map)[row_index])
+            {
+                continue;
+            }
+            auto str = col_str->getDataAt(row_index);
+            if (hasArabicIndicDigit(str))
+            {
+                has_arabic_indic_digit = true;
+                break;
+            }
+        }
+
+        if (!has_arabic_indic_digit)
+        {
+            // No Arabic indic digits found, return the original column
+            return arguments[0].column;
+        }
+
+        auto res_col = data_col->cloneEmpty();
+        if (row_index)
+        {
+            res_col->insertManyFrom(*data_col, 0, row_index);
+        }
+        for (; row_index < input_rows_count; ++row_index)
+        {
+            if (null_map && (*null_map)[row_index])
+            {
+                res_col->insertDefault();
+                continue;
+            }
+            auto str = convertArabicIndicDigit(col_str->getDataAt(row_index));
+            res_col->insertData(str.c_str(), str.size());
+        }
+        return res_col;
+    }
+
+private:
+    bool hasArabicIndicDigit(StringRef str) const
+    {
+        // In most cases, the first byte is a digit.
+        char c = reinterpret_cast<char>(str.data[0]);
+        if ('0' <= c && c <= '9')
+        {
+            return false;
+        }
+        return true;
+    }
+
+
+    bool isArabicIndicDigit(char32_t c) const { return c >= 0x0660 && c <= 0x0669; }
+    char toAsciiDigit(char32_t c) const { return static_cast<char>(c - 0x0660 + '0'); }
+
+    String convertArabicIndicDigit(const StringRef & str) const
+    {
+        std::string result;
+        result.reserve(str.size);
+        for (size_t i = 0; i < str.size;)
+        {
+            unsigned char c = str.data[i];
+            char32_t cp = 0;
+            if ((c & 0x80) == 0) // 1-byte
+            {
+                cp = c;
+                i += 1;
+            }
+            else if ((c & 0xE0) == 0xC0) // 2-byte
+            {
+                cp = ((c & 0x1F) << 6) | (str.data[i + 1] & 0x3F);
+                i += 2;
+            }
+            else if ((c & 0xF0) == 0xE0) // 3-byte
+            {
+                cp = ((c & 0x0F) << 12) | ((str.data[i + 1] & 0x3F) << 6) | (str.data[i + 2] & 0x3F);
+                i += 3;
+            }
+            else if ((c & 0xF8) == 0xF0) // 4-byte
+            {
+                cp = ((c & 0x07) << 18) | ((str.data[i + 1] & 0x3F) << 12) | ((str.data[i + 2] & 0x3F) << 6) | (str.data[i + 3] & 0x3F);
+                i += 4;
+            }
+            if (isArabicIndicDigit(cp))
+                result.push_back(toAsciiDigit(cp));
+            else
+                result.push_back(cp);
+        }
+        return result;
+    }
+};
+
+using namespace DB;
+REGISTER_FUNCTION(ArabicIndicToAsciiDigitForDate)
+{
+    factory.registerFunction<ArabicIndicToAsciiDigitForDateFunction>();
+}
+}
diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/getTimestamp.h b/cpp-ch/local-engine/Parser/scalar_function_parser/getTimestamp.h
@@ -61,7 +61,7 @@ class FunctionParserGetTimestamp : public FunctionParser
         auto parsed_args = parseFunctionArguments(substrait_func, actions_dag);
         if (parsed_args.size() != 2)
             throw DB::Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires exactly two arguments", getName());
-        const auto * expr_arg = parsed_args[0];
+        const auto * expr_arg = convertArabicIndicDigit(actions_dag, parsed_args[0]);
         const auto * fmt_arg = parsed_args[1];
         
         const auto & args = substrait_func.arguments();
@@ -129,5 +129,11 @@ class FunctionParserGetTimestamp : public FunctionParser
             return std::regex_match(fmt, fmtPattern);
         }
     }
+
+    const DB::ActionsDAG::Node * convertArabicIndicDigit(DB::ActionsDAG & actions_dag, const DB::ActionsDAG::Node * node) const
+    {
+        const auto * func_node = toFunctionNode(actions_dag, "arabic_indic_to_ascii_digit_for_date", {node});
+        return func_node;
+    }
 };
 }
diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/unixTimestamp.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/unixTimestamp.cpp
@@ -59,7 +59,6 @@ class FunctionParserUnixTimestamp : public FunctionParserGetTimestamp
             throw DB::Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires exactly two arguments", getName());
 
         const auto * expr_arg = parsed_args[0];
-        const auto * fmt_arg = parsed_args[1];
         auto expr_type = removeNullable(expr_arg->result_type);
         if (isString(expr_type))
             return FunctionParserGetTimestamp::parse(substrait_func, actions_dag);

Original file line number	Diff line number	Diff line change
`@@ -61,7 +61,7 @@ class FunctionParserGetTimestamp : public FunctionParser`
`61`	`61`	`auto parsed_args = parseFunctionArguments(substrait_func, actions_dag);`
`62`	`62`	`if (parsed_args.size() != 2)`
`63`	`63`	`throw DB::Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires exactly two arguments", getName());`
`64`		`- const auto * expr_arg = parsed_args[0];`
	`64`	`+ const auto * expr_arg = convertArabicIndicDigit(actions_dag, parsed_args[0]);`
`65`	`65`	`const auto * fmt_arg = parsed_args[1];`
`66`	`66`
`67`	`67`	`const auto & args = substrait_func.arguments();`
`@@ -129,5 +129,11 @@ class FunctionParserGetTimestamp : public FunctionParser`
`129`	`129`	`return std::regex_match(fmt, fmtPattern);`
`130`	`130`	`}`
`131`	`131`	`}`
	`132`	`+`
	`133`	`+ const DB::ActionsDAG::Node * convertArabicIndicDigit(DB::ActionsDAG & actions_dag, const DB::ActionsDAG::Node * node) const`
	`134`	`+ {`
	`135`	`+ const auto * func_node = toFunctionNode(actions_dag, "arabic_indic_to_ascii_digit_for_date", {node});`
	`136`	`+ return func_node;`
	`137`	`+ }`
`132`	`138`	`};`
`133`	`139`	`}`