Skip to content

Commit 6e88e73

Browse files
HTHouHTHou
committed
Merge branch 'force_ci/object_type' of github.com:apache/iotdb into add_base32_path
2 parents 431369a + 2bdba56 commit 6e88e73

File tree

9 files changed

+233
-13
lines changed

9 files changed

+233
-13
lines changed

‎integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@
4141
import org.junit.experimental.categories.Category;
4242
import org.junit.runner.RunWith;
4343

44+
import java.io.File;
45+
import java.nio.file.Files;
46+
import java.nio.file.Paths;
4447
import java.sql.Connection;
4548
import java.sql.ResultSet;
4649
import java.sql.ResultSetMetaData;
@@ -777,6 +780,136 @@ public void testConcurrentAutoCreateAndDropColumn() throws Exception {
777780
}
778781
}
779782

783+
@Test
784+
public void testTableObjectCheck() throws Exception {
785+
final Set<String> illegal = new HashSet<>(Arrays.asList("./", ".", "..", ".\\", "../hack"));
786+
for (final String single : illegal) {
787+
testObject4SingleIllegalPath(single);
788+
}
789+
}
790+
791+
private void testObject4SingleIllegalPath(final String illegal) throws Exception {
792+
try (final Connection connection =
793+
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
794+
final Statement statement = connection.createStatement();
795+
final ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
796+
statement.execute("create database if not exists db2");
797+
statement.execute("use db2");
798+
statement.execute(String.format("create table \"%s\" ()", illegal));
799+
800+
try {
801+
statement.execute(String.format("alter table \"%s\" add column a object", illegal));
802+
fail();
803+
} catch (final SQLException e) {
804+
Assert.assertEquals(
805+
String.format(
806+
"701: When there are object fields, the tableName %s shall not be '.', '..' or contain './', '.\\'",
807+
illegal),
808+
e.getMessage());
809+
}
810+
811+
// Test auto-create
812+
String testObject =
813+
System.getProperty("user.dir")
814+
+ File.separator
815+
+ "target"
816+
+ File.separator
817+
+ "test-classes"
818+
+ File.separator
819+
+ "ainode-example"
820+
+ File.separator
821+
+ "model.pt";
822+
823+
List<IMeasurementSchema> schemaList = new ArrayList<>();
824+
schemaList.add(new MeasurementSchema("a", TSDataType.STRING));
825+
schemaList.add(new MeasurementSchema("b", TSDataType.STRING));
826+
schemaList.add(new MeasurementSchema("c", TSDataType.INT32));
827+
schemaList.add(new MeasurementSchema(illegal, TSDataType.OBJECT));
828+
final List<ColumnCategory> columnTypes =
829+
Arrays.asList(
830+
ColumnCategory.TAG,
831+
ColumnCategory.ATTRIBUTE,
832+
ColumnCategory.FIELD,
833+
ColumnCategory.FIELD);
834+
final Tablet tablet =
835+
new Tablet(
836+
illegal,
837+
IMeasurementSchema.getMeasurementNameList(schemaList),
838+
IMeasurementSchema.getDataTypeList(schemaList),
839+
columnTypes,
840+
1);
841+
tablet.addTimestamp(0, System.currentTimeMillis());
842+
tablet.addValue(schemaList.get(0).getMeasurementName(), 0, "d1");
843+
tablet.addValue(schemaList.get(1).getMeasurementName(), 0, "a1");
844+
tablet.addValue(schemaList.get(2).getMeasurementName(), 0, 0);
845+
tablet.addValue(0, 3, true, 0, Files.readAllBytes(Paths.get(testObject)));
846+
847+
try {
848+
session.executeNonQueryStatement("use db2");
849+
session.insert(tablet);
850+
} catch (final Exception e) {
851+
Assert.assertEquals(
852+
String.format(
853+
"701: When there are object fields, the tableName %s shall not be '.', '..' or contain './', '.\\'",
854+
illegal),
855+
e.getMessage());
856+
}
857+
858+
try {
859+
statement.execute(String.format("create table test (\"%s\" object)", illegal));
860+
fail();
861+
} catch (final SQLException e) {
862+
Assert.assertEquals(
863+
String.format(
864+
"701: When there are object fields, the objectName %s shall not be '.', '..' or contain './', '.\\'",
865+
illegal),
866+
e.getMessage());
867+
}
868+
869+
statement.execute("create table test (a tag, b attribute, c int32, d object)");
870+
871+
// Cannot auto-extend illegal column
872+
tablet.setTableName("test");
873+
try {
874+
session.executeNonQueryStatement("use db2");
875+
session.insert(tablet);
876+
} catch (final Exception e) {
877+
Assert.assertEquals(
878+
String.format(
879+
"701: When there are object fields, the objectName %s shall not be '.', '..' or contain './', '.\\'",
880+
illegal),
881+
e.getMessage());
882+
}
883+
884+
// It's OK if you don't write object
885+
statement.execute(String.format("insert into test (a, b, c) values ('%s', 1, 1)", illegal));
886+
try {
887+
statement.execute(
888+
String.format("insert into test (a, b, c, d) values ('%s', 1, 1, 's')", illegal));
889+
fail();
890+
} catch (final SQLException e) {
891+
Assert.assertEquals(
892+
String.format(
893+
"507: When there are object fields, the deviceId [test, %s] shall not be '.', '..' or contain './', '.\\'",
894+
illegal),
895+
e.getMessage());
896+
}
897+
898+
try {
899+
statement.execute(String.format("alter table test add column \"%s\" object", illegal));
900+
fail();
901+
} catch (final SQLException e) {
902+
Assert.assertEquals(
903+
String.format(
904+
"701: When there are object fields, the objectName %s shall not be '.', '..' or contain './', '.\\'",
905+
illegal),
906+
e.getMessage());
907+
}
908+
909+
statement.execute("drop database db2");
910+
}
911+
}
912+
780913
@Test
781914
public void testTreeViewTable() throws Exception {
782915
try (final Connection connection = EnvFactory.getEnv().getConnection();

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@
111111
import org.apache.iotdb.rpc.TSStatusCode;
112112

113113
import org.apache.tsfile.annotations.TableModel;
114+
import org.apache.tsfile.enums.TSDataType;
114115
import org.apache.tsfile.file.metadata.IDeviceID;
115116
import org.apache.tsfile.utils.Pair;
116117
import org.slf4j.Logger;
@@ -124,6 +125,7 @@
124125
import java.util.Optional;
125126
import java.util.Set;
126127
import java.util.concurrent.ConcurrentHashMap;
128+
import java.util.concurrent.atomic.AtomicBoolean;
127129
import java.util.concurrent.locks.ReentrantLock;
128130
import java.util.stream.Collectors;
129131

@@ -1355,9 +1357,14 @@ public synchronized Pair<TSStatus, TsTable> tableColumnCheckForColumnExtension(
13551357
columnSchemaList.stream()
13561358
.map(TsTableColumnSchema::getColumnName)
13571359
.collect(Collectors.joining(", ")));
1360+
1361+
final AtomicBoolean hasObject = new AtomicBoolean(false);
13581362
columnSchemaList.removeIf(
13591363
columnSchema -> {
13601364
if (Objects.isNull(originalTable.getColumnSchema(columnSchema.getColumnName()))) {
1365+
if (columnSchema.getDataType().equals(TSDataType.OBJECT)) {
1366+
hasObject.set(true);
1367+
}
13611368
expandedTable.addColumnSchema(columnSchema);
13621369
return false;
13631370
}
@@ -1367,7 +1374,11 @@ public synchronized Pair<TSStatus, TsTable> tableColumnCheckForColumnExtension(
13671374
if (columnSchemaList.isEmpty()) {
13681375
return new Pair<>(RpcUtils.getStatus(TSStatusCode.COLUMN_ALREADY_EXISTS, errorMsg), null);
13691376
}
1370-
return new Pair<>(RpcUtils.SUCCESS_STATUS, expandedTable);
1377+
1378+
if (hasObject.get()) {
1379+
expandedTable.checkTableNameAndObjectNames4Object();
1380+
}
1381+
return new Pair<>(StatusUtils.OK, expandedTable);
13711382
}
13721383

13731384
public synchronized Pair<TSStatus, TsTable> tableColumnCheckForColumnRenaming(

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.iotdb.commons.auth.entity.PrivilegeType;
2828
import org.apache.iotdb.commons.auth.entity.User;
2929
import org.apache.iotdb.commons.exception.IllegalPathException;
30+
import org.apache.iotdb.commons.exception.MetadataException;
3031
import org.apache.iotdb.commons.exception.auth.AccessDeniedException;
3132
import org.apache.iotdb.commons.executable.ExecutableManager;
3233
import org.apache.iotdb.commons.path.PartialPath;
@@ -562,10 +563,12 @@ private Pair<String, TsTable> parseTable4CreateTableOrView(
562563
// TODO: Place the check at statement analyzer
563564
boolean hasTimeColumn = false;
564565
final Set<String> sourceNameSet = new HashSet<>();
566+
boolean hasObject = false;
565567
for (final ColumnDefinition columnDefinition : node.getElements()) {
566568
final TsTableColumnCategory category = columnDefinition.getColumnCategory();
567569
final String columnName = columnDefinition.getName().getValue();
568570
final TSDataType dataType = getDataType(columnDefinition.getType());
571+
hasObject |= dataType.equals(TSDataType.OBJECT);
569572
final String comment = columnDefinition.getComment();
570573
if (checkTimeColumnIdempotent(category, columnName, dataType, comment, table)
571574
&& !hasTimeColumn) {
@@ -594,6 +597,13 @@ private Pair<String, TsTable> parseTable4CreateTableOrView(
594597
}
595598
table.addColumnSchema(schema);
596599
}
600+
if (hasObject) {
601+
try {
602+
table.checkTableNameAndObjectNames4Object();
603+
} catch (final MetadataException e) {
604+
throw new SemanticException(e.getMessage(), e.getErrorCode());
605+
}
606+
}
597607
return new Pair<>(database, table);
598608
}
599609

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@
1919

2020
package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher;
2121

22-
import org.apache.iotdb.commons.exception.IoTDBException;
23-
import org.apache.iotdb.db.conf.IoTDBConfig;
24-
import org.apache.iotdb.db.conf.IoTDBDescriptor;
22+
import org.apache.iotdb.commons.exception.IoTDBRuntimeException;
23+
import org.apache.iotdb.commons.schema.table.TsTable;
24+
import org.apache.iotdb.db.exception.sql.SemanticException;
2525
import org.apache.iotdb.db.protocol.session.SessionManager;
2626
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
2727
import org.apache.iotdb.db.queryengine.plan.Coordinator;
@@ -53,8 +53,6 @@ public class TableDeviceSchemaValidator {
5353

5454
private static final Logger LOGGER = LoggerFactory.getLogger(TableDeviceSchemaValidator.class);
5555

56-
private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
57-
5856
private final Coordinator coordinator = Coordinator.getInstance();
5957

6058
private final TableDeviceSchemaFetcher fetcher = TableDeviceSchemaFetcher.getInstance();
@@ -247,9 +245,18 @@ private void autoCreateOrUpdateDeviceSchema(
247245
Long.MAX_VALUE,
248246
false);
249247
if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
250-
throw new RuntimeException(
251-
new IoTDBException(
252-
executionResult.status.getMessage(), executionResult.status.getCode()));
248+
throw new IoTDBRuntimeException(
249+
executionResult.status.getMessage(), executionResult.status.getCode());
250+
}
251+
}
252+
253+
public static void checkObject4DeviceId(final Object[] deviceId) {
254+
for (final Object part : deviceId) {
255+
final String value = (String) part;
256+
if (Objects.nonNull(value) && TsTable.isInvalid4ObjectType(value)) {
257+
throw new SemanticException(
258+
TsTable.getObjectStringError("deviceId", Arrays.toString(deviceId)));
259+
}
253260
}
254261
}
255262

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRow.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,15 @@
2121

2222
import org.apache.iotdb.db.exception.query.QueryProcessException;
2323
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
24+
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaValidator;
2425
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement;
2526

27+
import org.apache.tsfile.enums.TSDataType;
28+
2629
import java.util.Arrays;
2730
import java.util.Collections;
2831
import java.util.List;
32+
import java.util.Objects;
2933

3034
public class InsertRow extends WrappedInsertStatement {
3135

@@ -56,7 +60,12 @@ public String getTableName() {
5660
@Override
5761
public List<Object[]> getDeviceIdList() {
5862
final InsertRowStatement insertRowStatement = getInnerTreeStatement();
59-
Object[] segments = insertRowStatement.getTableDeviceID().getSegments();
63+
final Object[] segments = insertRowStatement.getTableDeviceID().getSegments();
64+
if (Objects.nonNull(getInnerTreeStatement().getMeasurementSchemas())
65+
&& Arrays.stream(getInnerTreeStatement().getMeasurementSchemas())
66+
.anyMatch(schema -> Objects.nonNull(schema) && schema.getType() == TSDataType.OBJECT)) {
67+
TableDeviceSchemaValidator.checkObject4DeviceId(segments);
68+
}
6069
return Collections.singletonList(Arrays.copyOfRange(segments, 1, segments.length));
6170
}
6271

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,17 @@
2525
import org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeUtils;
2626
import org.apache.iotdb.db.queryengine.plan.relational.metadata.ITableDeviceSchemaValidation;
2727
import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
28+
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaValidator;
2829
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement;
2930
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement;
3031

32+
import org.apache.tsfile.enums.TSDataType;
33+
3134
import java.util.ArrayList;
3235
import java.util.Arrays;
3336
import java.util.Collections;
3437
import java.util.List;
38+
import java.util.Objects;
3539

3640
public class InsertRows extends WrappedInsertStatement {
3741

@@ -117,8 +121,14 @@ public String getTableName() {
117121

118122
@Override
119123
public List<Object[]> getDeviceIdList() {
120-
Object[] idSegments = insertRowStatement.getTableDeviceID().getSegments();
121-
return Collections.singletonList(Arrays.copyOfRange(idSegments, 1, idSegments.length));
124+
final Object[] tagSegments = insertRowStatement.getTableDeviceID().getSegments();
125+
if (Objects.nonNull(insertRowStatement.getMeasurementSchemas())
126+
&& Arrays.stream(insertRowStatement.getMeasurementSchemas())
127+
.anyMatch(
128+
schema -> Objects.nonNull(schema) && schema.getType() == TSDataType.OBJECT)) {
129+
TableDeviceSchemaValidator.checkObject4DeviceId(tagSegments);
130+
}
131+
return Collections.singletonList(Arrays.copyOfRange(tagSegments, 1, tagSegments.length));
122132
}
123133

124134
@Override

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,18 @@
2121

2222
import org.apache.iotdb.db.exception.query.QueryProcessException;
2323
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
24+
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaValidator;
2425
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
2526

27+
import org.apache.tsfile.enums.TSDataType;
2628
import org.apache.tsfile.file.metadata.IDeviceID;
2729

2830
import java.util.ArrayList;
2931
import java.util.Arrays;
3032
import java.util.LinkedHashMap;
3133
import java.util.List;
3234
import java.util.Map;
35+
import java.util.Objects;
3336

3437
public class InsertTablet extends WrappedInsertStatement {
3538

@@ -65,6 +68,12 @@ public List<Object[]> getDeviceIdList() {
6568
List<Object[]> deviceIdList = new ArrayList<>();
6669
for (IDeviceID deviceID : deviceID2LastIdxMap.keySet()) {
6770
Object[] segments = deviceID.getSegments();
71+
if (Objects.nonNull(super.getInnerTreeStatement().getMeasurementSchemas())
72+
&& Arrays.stream(super.getInnerTreeStatement().getMeasurementSchemas())
73+
.anyMatch(
74+
schema -> Objects.nonNull(schema) && schema.getType() == TSDataType.OBJECT)) {
75+
TableDeviceSchemaValidator.checkObject4DeviceId(segments);
76+
}
6877
deviceIdList.add(Arrays.copyOfRange(segments, 1, segments.length));
6978
}
7079
return deviceIdList;

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1548,7 +1548,7 @@ public ISchemaReader<IDeviceSchemaInfo> getTableDeviceReader(final PartialPath p
15481548

15491549
@Override
15501550
public ISchemaReader<IDeviceSchemaInfo> getTableDeviceReader(
1551-
String table, List<Object[]> devicePathList) throws MetadataException {
1551+
String table, List<Object[]> devicePathList) {
15521552
throw new UnsupportedOperationException();
15531553
}
15541554

0 commit comments

Comments
 (0)