Supported "need last cache" semantic for table/database schema

integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java

@@ -109,6 +109,7 @@ public void testManageDatabase() {
 
       statement.execute("alter database if exists test1 set properties ttl='INF'");
       statement.execute("alter database test set properties ttl=default");
+      statement.execute("alter database test set properties need_last_cache=false");
 
       String[] databaseNames = new String[] {"test"};
       String[] TTLs = new String[] {"INF"};
@@ -160,6 +161,7 @@ public void testManageDatabase() {
           assertEquals(timePartitionInterval[cnt], resultSet.getLong(5));
           assertEquals(schemaRegionGroupNum[cnt], resultSet.getInt(6));
           assertEquals(dataRegionGroupNum[cnt], resultSet.getInt(7));
+          assertFalse(resultSet.getBoolean(8));
           cnt++;
         }
         assertEquals(databaseNames.length, cnt);
@@ -424,7 +426,8 @@ public void testInformationSchema() throws SQLException {
                   "data_replication_factor,INT32,ATTRIBUTE,",
                   "time_partition_interval,INT64,ATTRIBUTE,",
                   "schema_region_group_num,INT32,ATTRIBUTE,",
-                  "data_region_group_num,INT32,ATTRIBUTE,")));
+                  "data_region_group_num,INT32,ATTRIBUTE,",
+                  "need_last_cache,BOOLEAN,ATTRIBUTE,")));
       TestUtils.assertResultSetEqual(
           statement.executeQuery("desc tables"),
           "ColumnName,DataType,Category,",
@@ -435,7 +438,8 @@ public void testInformationSchema() throws SQLException {
                   "ttl(ms),STRING,ATTRIBUTE,",
                   "status,STRING,ATTRIBUTE,",
                   "comment,STRING,ATTRIBUTE,",
-                  "table_type,STRING,ATTRIBUTE,")));
+                  "table_type,STRING,ATTRIBUTE,",
+                  "need_last_cache,BOOLEAN,ATTRIBUTE,")));
       TestUtils.assertResultSetEqual(
           statement.executeQuery("desc columns"),
           "ColumnName,DataType,Category,",
@@ -603,39 +607,40 @@ public void testInformationSchema() throws SQLException {
       statement.execute(
           "create table test.test (a tag, b attribute, c int32 comment 'turbine') comment 'test'");
       statement.execute(
-          "CREATE VIEW test.view_table (tag1 STRING TAG,tag2 STRING TAG,s11 INT32 FIELD,s3 INT32 FIELD FROM s2) RESTRICT WITH (ttl=100) AS root.\"a\".**");
+          "CREATE VIEW test.view_table (tag1 STRING TAG,tag2 STRING TAG,s11 INT32 FIELD,s3 INT32 FIELD FROM s2) RESTRICT WITH (ttl=100, need_last_cache=true) AS root.\"a\".**");
 
       TestUtils.assertResultSetEqual(
           statement.executeQuery("select * from databases"),
-          "database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_interval,schema_region_group_num,data_region_group_num,",
+          "database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_interval,schema_region_group_num,data_region_group_num,need_last_cache,",
           new HashSet<>(
               Arrays.asList(
-                  "information_schema,INF,null,null,null,null,null,",
-                  "test,INF,1,1,604800000,0,0,")));
+                  "information_schema,INF,null,null,null,null,null,false,",
+                  "test,INF,1,1,604800000,0,0,true,")));
       TestUtils.assertResultSetEqual(
           statement.executeQuery("show devices from tables where status = 'USING'"),
-          "database,table_name,ttl(ms),status,comment,table_type,",
+          "database,table_name,ttl(ms),status,comment,table_type,need_last_cache,",
           new HashSet<>(
               Arrays.asList(
-                  "information_schema,databases,INF,USING,null,SYSTEM VIEW,",
-                  "information_schema,tables,INF,USING,null,SYSTEM VIEW,",
-                  "information_schema,columns,INF,USING,null,SYSTEM VIEW,",
-                  "information_schema,queries,INF,USING,null,SYSTEM VIEW,",
-                  "information_schema,regions,INF,USING,null,SYSTEM VIEW,",
-                  "information_schema,topics,INF,USING,null,SYSTEM VIEW,",
-                  "information_schema,pipe_plugins,INF,USING,null,SYSTEM VIEW,",
-                  "information_schema,pipes,INF,USING,null,SYSTEM VIEW,",
-                  "information_schema,subscriptions,INF,USING,null,SYSTEM VIEW,",
-                  "information_schema,views,INF,USING,null,SYSTEM VIEW,",
-                  "information_schema,functions,INF,USING,null,SYSTEM VIEW,",
-                  "information_schema,configurations,INF,USING,null,SYSTEM VIEW,",
-                  "information_schema,keywords,INF,USING,null,SYSTEM VIEW,",
-                  "information_schema,nodes,INF,USING,null,SYSTEM VIEW,",
-                  "information_schema,config_nodes,INF,USING,null,SYSTEM VIEW,",
-                  "information_schema,data_nodes,INF,USING,null,SYSTEM VIEW,",
+                  "information_schema,databases,INF,USING,null,SYSTEM VIEW,false,",
+                  "information_schema,tables,INF,USING,null,SYSTEM VIEW,false,",
+                  "information_schema,columns,INF,USING,null,SYSTEM VIEW,false,",
+                  "information_schema,queries,INF,USING,null,SYSTEM VIEW,false,",
+                  "information_schema,regions,INF,USING,null,SYSTEM VIEW,false,",
+                  "information_schema,topics,INF,USING,null,SYSTEM VIEW,false,",
+                  "information_schema,pipe_plugins,INF,USING,null,SYSTEM VIEW,false,",
+                  "information_schema,pipes,INF,USING,null,SYSTEM VIEW,false,",
+                  "information_schema,subscriptions,INF,USING,null,SYSTEM VIEW,false,",
+                  "information_schema,views,INF,USING,null,SYSTEM VIEW,false,",
+                  "information_schema,models,INF,USING,null,SYSTEM VIEW,false,",
+                  "information_schema,functions,INF,USING,null,SYSTEM VIEW,false,",
+                  "information_schema,configurations,INF,USING,null,SYSTEM VIEW,false,",
+                  "information_schema,keywords,INF,USING,null,SYSTEM VIEW,false,",
+                  "information_schema,nodes,INF,USING,null,SYSTEM VIEW,false,",
+                  "information_schema,config_nodes,INF,USING,null,SYSTEM VIEW,false,",
+                  "information_schema,data_nodes,INF,USING,null,SYSTEM VIEW,false,",
                   "information_schema,connections,INF,USING,null,SYSTEM VIEW,",
-                  "test,test,INF,USING,test,BASE TABLE,",
-                  "test,view_table,100,USING,null,VIEW FROM TREE,")));
+                  "test,test,INF,USING,test,BASE TABLE,true,",
+                  "test,view_table,100,USING,null,VIEW FROM TREE,true,")));
       TestUtils.assertResultSetEqual(
           statement.executeQuery("count devices from tables where status = 'USING'"),
           "count(devices),",
@@ -686,7 +691,7 @@ public void testInformationSchema() throws SQLException {
           statement.executeQuery("select * from views"),
           "database,table_name,view_definition,",
           Collections.singleton(
-              "test,view_table,CREATE VIEW \"view_table\" (\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" INT32 FIELD FROM \"s2\") RESTRICT WITH (ttl=100) AS root.\"a\".**,"));
+              "test,view_table,CREATE VIEW \"view_table\" (\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" INT32 FIELD FROM \"s2\") RESTRICT WITH (ttl=100, need_last_cache=true) AS root.\"a\".**,"));
 
       TestUtils.assertResultSetEqual(
           statement.executeQuery(
@@ -814,8 +819,8 @@ public void testDBAuth() throws SQLException {
           Collections.singleton("information_schema,INF,null,null,null,"));
       TestUtils.assertResultSetEqual(
           userStmt.executeQuery("select * from information_schema.databases"),
-          "database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_interval,schema_region_group_num,data_region_group_num,",
-          Collections.singleton("information_schema,INF,null,null,null,null,null,"));
+          "database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_interval,schema_region_group_num,data_region_group_num,need_last_cache,",
+          Collections.singleton("information_schema,INF,null,null,null,null,null,false,"));
     }
 
     try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);

integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java

@@ -126,6 +126,7 @@ public void testManageTable() {
       String[] ttls = new String[] {"INF"};
       String[] statuses = new String[] {"USING"};
       String[] comments = new String[] {"test"};
+      String[] needLastCaches = new String[] {"true"};
 
       statement.execute("use test2");
 
@@ -145,6 +146,7 @@ public void testManageTable() {
           assertEquals(ttls[cnt], resultSet.getString(2));
           assertEquals(statuses[cnt], resultSet.getString(3));
           assertEquals(comments[cnt], resultSet.getString(4));
+          assertEquals(needLastCaches[cnt], resultSet.getString(6));
           cnt++;
         }
         assertEquals(tableNames.length, cnt);
@@ -189,6 +191,9 @@ public void testManageTable() {
 
       statement.execute("comment on table test1.table1 is 'new_test'");
       comments = new String[] {"new_test"};
+
+      statement.execute("alter table test1.table1 set properties need_last_cache=false");
+      needLastCaches = new String[] {"false"};
       // using SHOW tables from
       try (final ResultSet resultSet = statement.executeQuery("SHOW tables details from test1")) {
         int cnt = 0;
@@ -202,6 +207,7 @@ public void testManageTable() {
           assertEquals(tableNames[cnt], resultSet.getString(1));
           assertEquals(ttls[cnt], resultSet.getString(2));
           assertEquals(comments[cnt], resultSet.getString(4));
+          assertEquals(needLastCaches[cnt], resultSet.getString(6));
           cnt++;
         }
         assertEquals(tableNames.length, cnt);
@@ -305,7 +311,7 @@ public void testManageTable() {
           statement.executeQuery("show create table table2"),
           "Table,Create Table,",
           Collections.singleton(
-              "table2,CREATE TABLE \"table2\" (\"region_id\" STRING TAG,\"plant_id\" STRING TAG,\"color\" STRING ATTRIBUTE,\"temperature\" FLOAT FIELD,\"speed\" DOUBLE FIELD COMMENT 'fast') WITH (ttl=6600000),"));
+              "table2,CREATE TABLE \"table2\" (\"region_id\" STRING TAG,\"plant_id\" STRING TAG,\"color\" STRING ATTRIBUTE,\"temperature\" FLOAT FIELD,\"speed\" DOUBLE FIELD COMMENT 'fast') WITH (ttl=6600000, need_last_cache=true),"));
 
       try {
         statement.execute("alter table table2 add column speed DOUBLE FIELD");
@@ -656,7 +662,7 @@ public void testTableAuth() throws SQLException {
       Assert.assertThrows(SQLException.class, () -> userStmt.execute("select * from db.test"));
       TestUtils.assertResultSetEqual(
           userStmt.executeQuery("select * from information_schema.tables where database = 'db'"),
-          "database,table_name,ttl(ms),status,comment,table_type,",
+          "database,table_name,ttl(ms),status,comment,table_type,need_last_cache,",
           Collections.emptySet());
       TestUtils.assertResultSetEqual(
           userStmt.executeQuery("select * from information_schema.columns where database = 'db'"),
@@ -939,8 +945,8 @@ public void testTreeViewTable() throws Exception {
 
       TestUtils.assertResultSetEqual(
           statement.executeQuery("show tables details"),
-          "TableName,TTL(ms),Status,Comment,TableType,",
-          Collections.singleton("view_table,100,USING,comment,VIEW FROM TREE,"));
+          "TableName,TTL(ms),Status,Comment,TableType,NeedLastCache,",
+          Collections.singleton("view_table,100,USING,comment,VIEW FROM TREE,true,"));
 
       TestUtils.assertResultSetEqual(
           statement.executeQuery("desc view_table"),
@@ -1075,14 +1081,14 @@ public void testTreeViewTable() throws Exception {
           statement.executeQuery("show create view view_table"),
           "View,Create View,",
           Collections.singleton(
-              "view_table,CREATE VIEW \"view_table\" (\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100) AS root.\"重庆\".**,"));
+              "view_table,CREATE VIEW \"view_table\" (\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100, need_last_cache=true) AS root.\"重庆\".**,"));
 
       // Can also use "show create table"
       TestUtils.assertResultSetEqual(
           statement.executeQuery("show create table view_table"),
           "View,Create View,",
           Collections.singleton(
-              "view_table,CREATE VIEW \"view_table\" (\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100) AS root.\"重庆\".**,"));
+              "view_table,CREATE VIEW \"view_table\" (\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100, need_last_cache=true) AS root.\"重庆\".**,"));
 
       statement.execute("create table a ()");
       try {

iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4

@@ -153,6 +153,7 @@ keyWords
     | MODELS
     | MODIFY
     | NAN
+    | NEED_LAST_CACHE
     | NODEID
     | NODES
     | NONE

iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4

@@ -111,6 +111,7 @@ databaseAttributesClause
 
 databaseAttributeClause
     : databaseAttributeKey operator_eq INTEGER_LITERAL
+    | NEED_LAST_CACHE operator_eq boolean_literal
     ;
 
 databaseAttributeKey

iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4

@@ -1194,6 +1194,10 @@ DATA_REGION_GROUP_NUM
     : D A T A '_' R E G I O N '_' G R O U P '_' N U M
     ;
 
+NEED_LAST_CACHE
+    : N E E D '_' L A S T '_' C A C H E
+    ;
+
 CURRENT_TIMESTAMP
     : C U R R E N T '_' T I M E S T A M P
     ;

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java

@@ -380,6 +380,7 @@ public TShowDatabaseResp showDatabase(final GetDatabasePlan getDatabasePlan) {
       databaseInfo.setDataReplicationFactor(databaseSchema.getDataReplicationFactor());
       databaseInfo.setTimePartitionOrigin(databaseSchema.getTimePartitionOrigin());
       databaseInfo.setTimePartitionInterval(databaseSchema.getTimePartitionInterval());
+      databaseInfo.setNeedLastCache(databaseSchema.isNeedLastCache());
       databaseInfo.setMinSchemaRegionNum(
           getMinRegionGroupNum(database, TConsensusGroupType.SchemaRegion));
       databaseInfo.setMaxSchemaRegionNum(
@@ -873,6 +874,10 @@ public static TSStatus enrichDatabaseSchemaWithDefaultProperties(
                   "Failed to create database. The timePartitionInterval should be positive.");
     }
 
+    if (!databaseSchema.isSetNeedLastCache()) {
+      databaseSchema.setNeedLastCache(true);
+    }
+
     if (isSystemDatabase || isAuditDatabase) {
       databaseSchema.setMinSchemaRegionGroupNum(1);
     } else if (!databaseSchema.isSetMinSchemaRegionGroupNum()) {

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java

@@ -136,6 +136,7 @@
 import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE_PATTERN;
 import static org.apache.iotdb.commons.schema.table.Audit.TABLE_MODEL_AUDIT_DATABASE;
 import static org.apache.iotdb.commons.schema.table.Audit.TREE_MODEL_AUDIT_DATABASE;
+import static org.apache.iotdb.commons.schema.table.TsTable.NEED_LAST_CACHE_PROPERTY;
 import static org.apache.iotdb.commons.schema.table.TsTable.TTL_PROPERTY;
 
 /**
@@ -271,6 +272,21 @@ public TSStatus alterDatabase(final DatabaseSchemaPlan plan) {
             currentSchema.getTTL());
       }
 
+      if (alterSchema.isSetNeedLastCache()
+          && alterSchema.isNeedLastCache()
+              != (!currentSchema.isSetNeedLastCache() || currentSchema.isNeedLastCache())) {
+        if (!currentSchema.isIsTableModel()) {
+          result.setCode(TSStatusCode.SEMANTIC_ERROR.getStatusCode());
+          result.setMessage("The tree model database does not support alter need last cache now.");
+          return result;
+        }
+        currentSchema.setNeedLastCache(alterSchema.isNeedLastCache());
+        LOGGER.info(
+            "[SetNeedLastCache] The need last cache flag of Database: {} is adjusted to: {}",
+            currentSchema.getName(),
+            currentSchema.isNeedLastCache());
+      }
+
       mTree
           .getDatabaseNodeByDatabasePath(partialPathName)
           .getAsMNode()
@@ -1285,6 +1301,11 @@ public ShowTableResp showTables(final ShowTablePlan plan) {
                             TreeViewSchema.isTreeViewTable(pair.getLeft())
                                 ? TableType.VIEW_FROM_TREE.ordinal()
                                 : TableType.BASE_TABLE.ordinal());
+                        info.setNeedLastCache(
+                            pair.getLeft()
+                                .getPropValue(NEED_LAST_CACHE_PROPERTY)
+                                .map(Boolean::parseBoolean)
+                                .orElse(true));
                         return info;
                       })
                   .collect(Collectors.toList())
@@ -1333,6 +1354,11 @@ public ShowTable4InformationSchemaResp showTables4InformationSchema() {
                                         TreeViewSchema.isTreeViewTable(pair.getLeft())
                                             ? TableType.VIEW_FROM_TREE.ordinal()
                                             : TableType.BASE_TABLE.ordinal());
+                                    info.setNeedLastCache(
+                                        pair.getLeft()
+                                            .getPropValue(NEED_LAST_CACHE_PROPERTY)
+                                            .map(Boolean::parseBoolean)
+                                            .orElse(true));
                                     return info;
                                   })
                               .collect(Collectors.toList()))));

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java

@@ -945,6 +945,11 @@ public void setTableProperties(
               && databaseNode.getDatabaseSchema().isSetTTL()
               && databaseNode.getDatabaseSchema().getTTL() != Long.MAX_VALUE) {
             table.addProp(k, String.valueOf(databaseNode.getDatabaseSchema().getTTL()));
+          } else if (k.equals(TsTable.NEED_LAST_CACHE_PROPERTY)
+              && databaseNode.getDatabaseSchema().isSetNeedLastCache()) {
+            table.addProp(
+                TsTable.NEED_LAST_CACHE_PROPERTY,
+                String.valueOf(databaseNode.getDatabaseSchema().isNeedLastCache()));
           } else {
             table.removeProp(k);
           }

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java

@@ -129,6 +129,10 @@ protected void checkTableExistence(final ConfigNodeProcedureEnv env) {
             && schema.getTTL() != Long.MAX_VALUE) {
           table.addProp(TsTable.TTL_PROPERTY, String.valueOf(schema.getTTL()));
         }
+        if (!table.getPropValue(TsTable.NEED_LAST_CACHE_PROPERTY).isPresent()
+            && schema.isSetNeedLastCache()) {
+          table.addProp(TsTable.NEED_LAST_CACHE_PROPERTY, String.valueOf(schema.isNeedLastCache()));
+        }
         setNextState(CreateTableState.PRE_CREATE);
       }
     } catch (final MetadataException | DatabaseNotExistsException e) {

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java

@@ -101,6 +101,11 @@ protected void checkTableExistence(final ConfigNodeProcedureEnv env) {
               && schema.getTTL() != Long.MAX_VALUE) {
             table.addProp(TsTable.TTL_PROPERTY, String.valueOf(schema.getTTL()));
           }
+          if (!table.getPropValue(TsTable.NEED_LAST_CACHE_PROPERTY).isPresent()
+              && schema.isSetNeedLastCache()) {
+            table.addProp(
+                TsTable.NEED_LAST_CACHE_PROPERTY, String.valueOf(schema.isNeedLastCache()));
+          }
           setNextState(CreateTableState.PRE_CREATE);
         }
       } catch (final MetadataException | DatabaseNotExistsException e) {