Skip to content

Commit 85281fa

Browse files
alpass163alpass163
authored
added support for Blob objects to the length() function, allowing for calculating the byte size for Blob inputs (#16170)
1 parent 85b8f16 commit 85281fa

File tree

6 files changed

+341
-18
lines changed

6 files changed

+341
-18
lines changed

‎integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBScalarFunctionTableIT.java‎

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1370,63 +1370,56 @@ public void lengthTestFail() {
13701370
tableAssertTestFail(
13711371
"select s1,Length(s1,1) from lengthTable",
13721372
TSStatusCode.SEMANTIC_ERROR.getStatusCode()
1373-
+ ": Scalar function length only accepts one argument and it must be text or string data type.",
1373+
+ ": Scalar function length only accepts one argument and it must be text, string, or blob data type.",
13741374
DATABASE_NAME);
13751375

13761376
// case 2: wrong data type
13771377
tableAssertTestFail(
13781378
"select s1,Length(s2) from lengthTable",
13791379
TSStatusCode.SEMANTIC_ERROR.getStatusCode()
1380-
+ ": Scalar function length only accepts one argument and it must be text or string data type.",
1380+
+ ": Scalar function length only accepts one argument and it must be text, string, or blob data type.",
13811381
DATABASE_NAME);
13821382

13831383
// case 3: wrong data type
13841384
tableAssertTestFail(
13851385
"select s1,Length(s3) from lengthTable",
13861386
TSStatusCode.SEMANTIC_ERROR.getStatusCode()
1387-
+ ": Scalar function length only accepts one argument and it must be text or string data type.",
1387+
+ ": Scalar function length only accepts one argument and it must be text, string, or blob data type.",
13881388
DATABASE_NAME);
13891389

13901390
// case 4: wrong data type
13911391
tableAssertTestFail(
13921392
"select s1,Length(s4) from lengthTable",
13931393
TSStatusCode.SEMANTIC_ERROR.getStatusCode()
1394-
+ ": Scalar function length only accepts one argument and it must be text or string data type.",
1394+
+ ": Scalar function length only accepts one argument and it must be text, string, or blob data type.",
13951395
DATABASE_NAME);
13961396

13971397
// case 5: wrong data type
13981398
tableAssertTestFail(
13991399
"select s1,Length(s5) from lengthTable",
14001400
TSStatusCode.SEMANTIC_ERROR.getStatusCode()
1401-
+ ": Scalar function length only accepts one argument and it must be text or string data type.",
1401+
+ ": Scalar function length only accepts one argument and it must be text, string, or blob data type.",
14021402
DATABASE_NAME);
14031403

14041404
// case 6: wrong data type
14051405
tableAssertTestFail(
14061406
"select s1,Length(s6) from lengthTable",
14071407
TSStatusCode.SEMANTIC_ERROR.getStatusCode()
1408-
+ ": Scalar function length only accepts one argument and it must be text or string data type.",
1408+
+ ": Scalar function length only accepts one argument and it must be text, string, or blob data type.",
14091409
DATABASE_NAME);
14101410

14111411
// case 7: wrong data type
14121412
tableAssertTestFail(
14131413
"select s1,Length(s7) from lengthTable",
14141414
TSStatusCode.SEMANTIC_ERROR.getStatusCode()
1415-
+ ": Scalar function length only accepts one argument and it must be text or string data type.",
1415+
+ ": Scalar function length only accepts one argument and it must be text, string, or blob data type.",
14161416
DATABASE_NAME);
14171417

14181418
// case 8: wrong data type
14191419
tableAssertTestFail(
14201420
"select s1,Length(s8) from lengthTable",
14211421
TSStatusCode.SEMANTIC_ERROR.getStatusCode()
1422-
+ ": Scalar function length only accepts one argument and it must be text or string data type.",
1423-
DATABASE_NAME);
1424-
1425-
// case 9: wrong data type
1426-
tableAssertTestFail(
1427-
"select s1,Length(s10) from lengthTable",
1428-
TSStatusCode.SEMANTIC_ERROR.getStatusCode()
1429-
+ ": Scalar function length only accepts one argument and it must be text or string data type.",
1422+
+ ": Scalar function length only accepts one argument and it must be text, string, or blob data type.",
14301423
DATABASE_NAME);
14311424
}
14321425

‎integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBLengthFunctionIT.java‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.relational.it.query.recent;
21+
22+
import org.apache.iotdb.it.env.EnvFactory;
23+
import org.apache.iotdb.it.framework.IoTDBTestRunner;
24+
import org.apache.iotdb.itbase.category.TableClusterIT;
25+
import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
26+
import org.apache.iotdb.rpc.TSStatusCode;
27+
28+
import org.junit.AfterClass;
29+
import org.junit.BeforeClass;
30+
import org.junit.Test;
31+
import org.junit.experimental.categories.Category;
32+
import org.junit.runner.RunWith;
33+
34+
import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData;
35+
import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail;
36+
import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest;
37+
38+
@RunWith(IoTDBTestRunner.class)
39+
@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
40+
public class IoTDBLengthFunctionIT {
41+
42+
private static final String DATABASE_NAME = "test_length_function";
43+
private static final String[] createSqls =
44+
new String[] {
45+
"CREATE DATABASE " + DATABASE_NAME,
46+
"USE " + DATABASE_NAME,
47+
"CREATE TABLE table1(c_text TEXT FIELD, c_string STRING FIELD, c_blob BLOB FIELD, c_int INT32 FIELD)",
48+
"INSERT INTO table1(time, c_text, c_string, c_blob) VALUES (1, 'hello', 'hello', X'68656C6C6F')",
49+
"INSERT INTO table1(time, c_text, c_string, c_blob) VALUES (2, '你好', '你好', X'e4bda0e5a5bd')",
50+
"INSERT INTO table1(time, c_text, c_string, c_blob) VALUES (3, '', '', X'')",
51+
"INSERT INTO table1(time, c_int) VALUES (4, 404)"
52+
};
53+
54+
@BeforeClass
55+
public static void setUp() throws Exception {
56+
EnvFactory.getEnv().initClusterEnvironment();
57+
prepareTableData(createSqls);
58+
}
59+
60+
@AfterClass
61+
public static void tearDown() throws Exception {
62+
EnvFactory.getEnv().cleanClusterEnvironment();
63+
}
64+
65+
/** validate LENGTH() for TEXT and STRING types, correctly calculate the character count */
66+
@Test
67+
public void testLengthOnTextAndString() {
68+
String[] expectedHeader = new String[] {"time", "length(c_text)", "length(c_string)"};
69+
String[] retArray =
70+
new String[] {
71+
"1970-01-01T00:00:00.001Z,5,5,",
72+
"1970-01-01T00:00:00.002Z,2,2,",
73+
"1970-01-01T00:00:00.003Z,0,0,",
74+
"1970-01-01T00:00:00.004Z,null,null,"
75+
};
76+
77+
tableResultSetEqualTest(
78+
"SELECT time, length(c_text) as \"length(c_text)\", length(c_string) as \"length(c_string)\" FROM table1",
79+
expectedHeader,
80+
retArray,
81+
DATABASE_NAME);
82+
}
83+
84+
/** validate LENGTH() for BLOB type, correctly calculate the number of bytes */
85+
@Test
86+
public void testLengthOnBlob() {
87+
String[] expectedHeader = new String[] {"time", "length(c_blob)"};
88+
String[] retArray =
89+
new String[] {
90+
"1970-01-01T00:00:00.001Z,5,",
91+
"1970-01-01T00:00:00.002Z,6,",
92+
"1970-01-01T00:00:00.003Z,0,",
93+
"1970-01-01T00:00:00.004Z,null,"
94+
};
95+
96+
tableResultSetEqualTest(
97+
"SELECT time, length(c_blob) as \"length(c_blob)\" FROM table1",
98+
expectedHeader,
99+
retArray,
100+
DATABASE_NAME);
101+
}
102+
103+
/**
104+
* Test the error handling behavior of the LENGTH() function when it receives invalid parameters.
105+
*/
106+
@Test
107+
public void testLengthFunctionOnInvalidInputs() {
108+
String expectedErrorMessage =
109+
TSStatusCode.SEMANTIC_ERROR.getStatusCode()
110+
+ ": Scalar function length only accepts one argument and it must be text, string, or blob data type.";
111+
112+
// Exception 1: Using LENGTH() on non-TEXT/BLOB/STRING types
113+
tableAssertTestFail("SELECT length(c_int) FROM table1", expectedErrorMessage, DATABASE_NAME);
114+
115+
// Exception 2: Incorrect number of arguments passed to the LENGTH() function
116+
tableAssertTestFail(
117+
"SELECT length(c_text, 1) FROM table1", expectedErrorMessage, DATABASE_NAME);
118+
}
119+
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@
128128
import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.BitwiseRightShiftColumnTransformer;
129129
import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.BitwiseXor2ColumnTransformer;
130130
import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.BitwiseXorColumnTransformer;
131+
import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.BlobLengthColumnTransformer;
131132
import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.BytesToDoubleColumnTransformer;
132133
import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.BytesToFloatColumnTransformer;
133134
import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.BytesToIntColumnTransformer;
@@ -775,7 +776,12 @@ private ColumnTransformer getFunctionColumnTransformer(
775776
} else if (TableBuiltinScalarFunction.LENGTH.getFunctionName().equalsIgnoreCase(functionName)) {
776777
ColumnTransformer first = this.process(children.get(0), context);
777778
if (children.size() == 1) {
778-
return new LengthColumnTransformer(INT32, first);
779+
Type argumentType = first.getType();
780+
if (isCharType(argumentType)) {
781+
return new LengthColumnTransformer(INT32, first);
782+
} else {
783+
return new BlobLengthColumnTransformer(INT32, first);
784+
}
779785
}
780786
} else if (TableBuiltinScalarFunction.UPPER.getFunctionName().equalsIgnoreCase(functionName)) {
781787
ColumnTransformer first = this.process(children.get(0), context);

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -265,11 +265,12 @@ && isIntegerNumber(argumentTypes.get(2)))) {
265265
}
266266
return STRING;
267267
} else if (TableBuiltinScalarFunction.LENGTH.getFunctionName().equalsIgnoreCase(functionName)) {
268-
if (!(argumentTypes.size() == 1 && isCharType(argumentTypes.get(0)))) {
268+
if (!(argumentTypes.size() == 1
269+
&& (isCharType(argumentTypes.get(0)) || isBlobType(argumentTypes.get(0))))) {
269270
throw new SemanticException(
270271
"Scalar function "
271272
+ functionName.toLowerCase(Locale.ENGLISH)
272-
+ " only accepts one argument and it must be text or string data type.");
273+
+ " only accepts one argument and it must be text, string, or blob data type.");
273274
}
274275
return INT32;
275276
} else if (TableBuiltinScalarFunction.UPPER.getFunctionName().equalsIgnoreCase(functionName)) {

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformer.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar;
21+
22+
import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer;
23+
import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.UnaryColumnTransformer;
24+
25+
import org.apache.tsfile.block.column.Column;
26+
import org.apache.tsfile.block.column.ColumnBuilder;
27+
import org.apache.tsfile.read.common.type.Type;
28+
import org.apache.tsfile.utils.Binary;
29+
30+
public class BlobLengthColumnTransformer extends UnaryColumnTransformer {
31+
32+
public BlobLengthColumnTransformer(Type returnType, ColumnTransformer childColumnTransformer) {
33+
super(returnType, childColumnTransformer);
34+
}
35+
36+
@Override
37+
protected void doTransform(Column column, ColumnBuilder columnBuilder) {
38+
doTransform(column, columnBuilder, null);
39+
}
40+
41+
@Override
42+
protected void doTransform(Column column, ColumnBuilder columnBuilder, boolean[] selection) {
43+
44+
int positionCount = column.getPositionCount();
45+
for (int i = 0; i < positionCount; i++) {
46+
if ((selection != null && !selection[i]) || column.isNull(i)) {
47+
columnBuilder.appendNull();
48+
continue;
49+
}
50+
51+
Binary value = column.getBinary(i);
52+
int length = value.getValues().length;
53+
columnBuilder.writeInt(length);
54+
}
55+
}
56+
}

0 commit comments

Comments
 (0)