Skip to content

Commit 97fa584

Browse files
Wei-hao-LiJackieTien97
Wei-hao-Li
authored and
JackieTien97
committed
Support system table current_queries and queries_costs_histogram (#16890)
(cherry picked from commit 94461b0)
1 parent 1a51743 commit 97fa584

File tree

24 files changed

+895
-24
lines changed

24 files changed

+895
-24
lines changed

‎integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,4 +137,10 @@ public DataNodeConfig setDataNodeMemoryProportion(String dataNodeMemoryProportio
137137
setProperty("datanode_memory_proportion", dataNodeMemoryProportion);
138138
return this;
139139
}
140+
141+
@Override
142+
public DataNodeConfig setQueryCostStatWindow(int queryCostStatWindow) {
143+
setProperty("query_cost_stat_window", String.valueOf(queryCostStatWindow));
144+
return this;
145+
}
140146
}

‎integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,4 +93,9 @@ public DataNodeConfig setDeleteWalFilesPeriodInMs(long deleteWalFilesPeriodInMs)
9393
public DataNodeConfig setDataNodeMemoryProportion(String dataNodeMemoryProportion) {
9494
return this;
9595
}
96+
97+
@Override
98+
public DataNodeConfig setQueryCostStatWindow(int queryCostStatWindow) {
99+
return this;
100+
}
96101
}

‎integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,4 +51,6 @@ DataNodeConfig setLoadTsFileAnalyzeSchemaMemorySizeInBytes(
5151
DataNodeConfig setDeleteWalFilesPeriodInMs(long deleteWalFilesPeriodInMs);
5252

5353
DataNodeConfig setDataNodeMemoryProportion(String dataNodeMemoryProportion);
54+
55+
DataNodeConfig setQueryCostStatWindow(int queryCostStatWindow);
5456
}

‎integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/informationschema/IoTDBCurrentQueriesIT.java‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.relational.it.query.recent.informationschema;
21+
22+
import org.apache.iotdb.commons.conf.CommonDescriptor;
23+
import org.apache.iotdb.db.queryengine.execution.QueryState;
24+
import org.apache.iotdb.it.env.EnvFactory;
25+
import org.apache.iotdb.it.framework.IoTDBTestRunner;
26+
import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
27+
import org.apache.iotdb.itbase.env.BaseEnv;
28+
29+
import org.junit.AfterClass;
30+
import org.junit.Assert;
31+
import org.junit.BeforeClass;
32+
import org.junit.Test;
33+
import org.junit.experimental.categories.Category;
34+
import org.junit.runner.RunWith;
35+
36+
import java.sql.Connection;
37+
import java.sql.ResultSet;
38+
import java.sql.ResultSetMetaData;
39+
import java.sql.SQLException;
40+
import java.sql.Statement;
41+
42+
import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.END_TIME_TABLE_MODEL;
43+
import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.NUMS;
44+
import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.STATEMENT_TABLE_MODEL;
45+
import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.STATE_TABLE_MODEL;
46+
import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.USER_TABLE_MODEL;
47+
import static org.apache.iotdb.commons.schema.table.InformationSchema.getSchemaTables;
48+
import static org.apache.iotdb.db.it.utils.TestUtils.createUser;
49+
import static org.apache.iotdb.itbase.env.BaseEnv.TABLE_SQL_DIALECT;
50+
import static org.junit.Assert.fail;
51+
52+
@RunWith(IoTDBTestRunner.class)
53+
@Category({TableLocalStandaloneIT.class})
54+
// This IT will run at least 60s, so we only run it in 1C1D
55+
public class IoTDBCurrentQueriesIT {
56+
private static final int CURRENT_QUERIES_COLUMN_NUM =
57+
getSchemaTables().get("current_queries").getColumnNum();
58+
private static final int QUERIES_COSTS_HISTOGRAM_COLUMN_NUM =
59+
getSchemaTables().get("queries_costs_histogram").getColumnNum();
60+
private static final String ADMIN_NAME =
61+
CommonDescriptor.getInstance().getConfig().getDefaultAdminName();
62+
private static final String ADMIN_PWD =
63+
CommonDescriptor.getInstance().getConfig().getAdminPassword();
64+
65+
@BeforeClass
66+
public static void setUp() throws Exception {
67+
EnvFactory.getEnv().getConfig().getDataNodeConfig().setQueryCostStatWindow(1);
68+
EnvFactory.getEnv().initClusterEnvironment();
69+
createUser("test", "test123123456");
70+
}
71+
72+
@AfterClass
73+
public static void tearDown() throws Exception {
74+
EnvFactory.getEnv().cleanClusterEnvironment();
75+
}
76+
77+
@Test
78+
public void testCurrentQueries() {
79+
try {
80+
Connection connection =
81+
EnvFactory.getEnv().getConnection(ADMIN_NAME, ADMIN_PWD, BaseEnv.TABLE_SQL_DIALECT);
82+
Statement statement = connection.createStatement();
83+
statement.execute("USE information_schema");
84+
85+
// 1. query current_queries table
86+
String sql = "SELECT * FROM current_queries";
87+
ResultSet resultSet = statement.executeQuery(sql);
88+
ResultSetMetaData metaData = resultSet.getMetaData();
89+
Assert.assertEquals(CURRENT_QUERIES_COLUMN_NUM, metaData.getColumnCount());
90+
int rowNum = 0;
91+
while (resultSet.next()) {
92+
Assert.assertEquals(QueryState.RUNNING.name(), resultSet.getString(STATE_TABLE_MODEL));
93+
Assert.assertEquals(null, resultSet.getString(END_TIME_TABLE_MODEL));
94+
Assert.assertEquals(sql, resultSet.getString(STATEMENT_TABLE_MODEL));
95+
Assert.assertEquals(ADMIN_NAME, resultSet.getString(USER_TABLE_MODEL));
96+
rowNum++;
97+
}
98+
Assert.assertEquals(1, rowNum);
99+
resultSet.close();
100+
101+
// 2. query queries_costs_histogram table
102+
sql = "SELECT * FROM queries_costs_histogram";
103+
resultSet = statement.executeQuery(sql);
104+
metaData = resultSet.getMetaData();
105+
Assert.assertEquals(QUERIES_COSTS_HISTOGRAM_COLUMN_NUM, metaData.getColumnCount());
106+
rowNum = 0;
107+
int queriesCount = 0;
108+
while (resultSet.next()) {
109+
int nums = resultSet.getInt(NUMS);
110+
if (nums > 0) {
111+
queriesCount++;
112+
}
113+
rowNum++;
114+
}
115+
Assert.assertEquals(1, queriesCount);
116+
Assert.assertEquals(61, rowNum);
117+
118+
// 3. requery current_queries table
119+
sql = "SELECT * FROM current_queries";
120+
resultSet = statement.executeQuery(sql);
121+
metaData = resultSet.getMetaData();
122+
Assert.assertEquals(CURRENT_QUERIES_COLUMN_NUM, metaData.getColumnCount());
123+
rowNum = 0;
124+
int finishedQueries = 0;
125+
while (resultSet.next()) {
126+
if (QueryState.FINISHED.name().equals(resultSet.getString(STATE_TABLE_MODEL))) {
127+
finishedQueries++;
128+
}
129+
rowNum++;
130+
}
131+
// three rows in the result, 2 FINISHED and 1 RUNNING
132+
Assert.assertEquals(3, rowNum);
133+
Assert.assertEquals(2, finishedQueries);
134+
resultSet.close();
135+
136+
// 4. test the expired QueryInfo was evicted
137+
Thread.sleep(61_001);
138+
resultSet = statement.executeQuery(sql);
139+
rowNum = 0;
140+
while (resultSet.next()) {
141+
rowNum++;
142+
}
143+
// one row in the result, current query
144+
Assert.assertEquals(1, rowNum);
145+
resultSet.close();
146+
147+
sql = "SELECT * FROM queries_costs_histogram";
148+
resultSet = statement.executeQuery(sql);
149+
queriesCount = 0;
150+
while (resultSet.next()) {
151+
int nums = resultSet.getInt(NUMS);
152+
if (nums > 0) {
153+
queriesCount++;
154+
}
155+
}
156+
// the last current_queries table query was recorded, others are evicted
157+
Assert.assertEquals(1, queriesCount);
158+
} catch (Exception e) {
159+
fail(e.getMessage());
160+
}
161+
162+
// 5. test privilege
163+
testPrivilege();
164+
}
165+
166+
private void testPrivilege() {
167+
// 1. test current_queries table
168+
try (Connection connection =
169+
EnvFactory.getEnv().getConnection("test", "test123123456", TABLE_SQL_DIALECT);
170+
Statement statement = connection.createStatement()) {
171+
String sql = "SELECT * FROM information_schema.current_queries";
172+
173+
// another user executes a query
174+
try (Connection connection2 =
175+
EnvFactory.getEnv().getConnection(ADMIN_NAME, ADMIN_PWD, BaseEnv.TABLE_SQL_DIALECT)) {
176+
ResultSet resultSet = connection2.createStatement().executeQuery(sql);
177+
resultSet.close();
178+
} catch (Exception e) {
179+
fail(e.getMessage());
180+
}
181+
182+
// current user query current_queries table
183+
ResultSet resultSet = statement.executeQuery(sql);
184+
int rowNum = 0;
185+
while (resultSet.next()) {
186+
rowNum++;
187+
}
188+
// only current query in the result
189+
Assert.assertEquals(1, rowNum);
190+
} catch (SQLException e) {
191+
fail(e.getMessage());
192+
}
193+
194+
// 2. test queries_costs_histogram table
195+
try (Connection connection =
196+
EnvFactory.getEnv().getConnection("test", "test123123456", TABLE_SQL_DIALECT);
197+
Statement statement = connection.createStatement()) {
198+
statement.executeQuery("SELECT * FROM information_schema.queries_costs_histogram");
199+
} catch (SQLException e) {
200+
Assert.assertEquals(
201+
"803: Access Denied: No permissions for this operation, please add privilege SYSTEM",
202+
e.getMessage());
203+
}
204+
}
205+
}

‎integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -399,6 +399,7 @@ public void testInformationSchema() throws SQLException {
399399
"config_nodes,INF,",
400400
"configurations,INF,",
401401
"connections,INF,",
402+
"current_queries,INF,",
402403
"data_nodes,INF,",
403404
"databases,INF,",
404405
"functions,INF,",
@@ -407,6 +408,7 @@ public void testInformationSchema() throws SQLException {
407408
"pipe_plugins,INF,",
408409
"pipes,INF,",
409410
"queries,INF,",
411+
"queries_costs_histogram,INF,",
410412
"regions,INF,",
411413
"subscriptions,INF,",
412414
"tables,INF,",
@@ -634,12 +636,14 @@ public void testInformationSchema() throws SQLException {
634636
"information_schema,config_nodes,INF,USING,null,SYSTEM VIEW,",
635637
"information_schema,data_nodes,INF,USING,null,SYSTEM VIEW,",
636638
"information_schema,connections,INF,USING,null,SYSTEM VIEW,",
639+
"information_schema,current_queries,INF,USING,null,SYSTEM VIEW,",
640+
"information_schema,queries_costs_histogram,INF,USING,null,SYSTEM VIEW,",
637641
"test,test,INF,USING,test,BASE TABLE,",
638642
"test,view_table,100,USING,null,VIEW FROM TREE,")));
639643
TestUtils.assertResultSetEqual(
640644
statement.executeQuery("count devices from tables where status = 'USING'"),
641645
"count(devices),",
642-
Collections.singleton("19,"));
646+
Collections.singleton("21,"));
643647
TestUtils.assertResultSetEqual(
644648
statement.executeQuery(
645649
"select * from columns where table_name = 'queries' or database = 'test'"),

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -814,6 +814,9 @@ public class IoTDBConfig {
814814
/** time cost(ms) threshold for slow query. Unit: millisecond */
815815
private long slowQueryThreshold = 10000;
816816

817+
/** time window threshold for record of history queries. Unit: minute */
818+
private int queryCostStatWindow = 0;
819+
817820
private int patternMatchingThreshold = 1000000;
818821

819822
/**
@@ -2628,6 +2631,14 @@ public void setSlowQueryThreshold(long slowQueryThreshold) {
26282631
this.slowQueryThreshold = slowQueryThreshold;
26292632
}
26302633

2634+
public int getQueryCostStatWindow() {
2635+
return queryCostStatWindow;
2636+
}
2637+
2638+
public void setQueryCostStatWindow(int queryCostStatWindow) {
2639+
this.queryCostStatWindow = queryCostStatWindow;
2640+
}
2641+
26312642
public boolean isEnableIndex() {
26322643
return enableIndex;
26332644
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -813,6 +813,11 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException
813813
properties.getProperty(
814814
"slow_query_threshold", String.valueOf(conf.getSlowQueryThreshold()))));
815815

816+
conf.setQueryCostStatWindow(
817+
Integer.parseInt(
818+
properties.getProperty(
819+
"query_cost_stat_window", String.valueOf(conf.getQueryCostStatWindow()))));
820+
816821
conf.setDataRegionNum(
817822
Integer.parseInt(
818823
properties.getProperty("data_region_num", String.valueOf(conf.getDataRegionNum()))));

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/impl/RestApiServiceImpl.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,8 @@ public Response executeFastLastQueryStatement(
212212
t = e;
213213
return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build();
214214
} finally {
215-
long costTime = System.nanoTime() - startTime;
215+
long endTime = System.nanoTime();
216+
long costTime = endTime - startTime;
216217

217218
StatementType statementType =
218219
Optional.ofNullable(statement)
@@ -227,7 +228,18 @@ public Response executeFastLastQueryStatement(
227228
if (queryId != null) {
228229
COORDINATOR.cleanupQueryExecution(queryId);
229230
} else {
230-
recordQueries(() -> costTime, new FastLastQueryContentSupplier(prefixPathList), t);
231+
IClientSession clientSession = SESSION_MANAGER.getCurrSession();
232+
233+
Supplier<String> contentOfQuerySupplier = new FastLastQueryContentSupplier(prefixPathList);
234+
COORDINATOR.recordCurrentQueries(
235+
null,
236+
startTime / 1_000_000,
237+
endTime / 1_000_000,
238+
costTime,
239+
contentOfQuerySupplier,
240+
clientSession.getUsername(),
241+
clientSession.getClientAddress());
242+
recordQueries(() -> costTime, contentOfQuerySupplier, t);
231243
}
232244
}
233245
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1050,13 +1050,23 @@ public TSExecuteStatementResp executeFastLastDataQueryForOnePrefixPath(
10501050

10511051
resp.setMoreData(false);
10521052

1053-
long costTime = System.nanoTime() - startTime;
1053+
long endTime = System.nanoTime();
1054+
long costTime = endTime - startTime;
10541055

10551056
CommonUtils.addStatementExecutionLatency(
10561057
OperationType.EXECUTE_QUERY_STATEMENT, StatementType.FAST_LAST_QUERY.name(), costTime);
10571058
CommonUtils.addQueryLatency(StatementType.FAST_LAST_QUERY, costTime);
1058-
recordQueries(
1059-
() -> costTime, () -> String.format("thrift fastLastQuery %s", prefixPath), null);
1059+
1060+
String statement = String.format("thrift fastLastQuery %s", prefixPath);
1061+
COORDINATOR.recordCurrentQueries(
1062+
null,
1063+
startTime / 1_000_000,
1064+
endTime / 1_000_000,
1065+
costTime,
1066+
() -> statement,
1067+
clientSession.getUsername(),
1068+
clientSession.getClientAddress());
1069+
recordQueries(() -> costTime, () -> statement, null);
10601070
return resp;
10611071
} catch (final Exception e) {
10621072
return RpcUtils.getTSExecuteStatementResp(

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/QueryId.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.apache.iotdb.db.queryengine.common;
2121

22+
import org.apache.iotdb.db.conf.IoTDBDescriptor;
2223
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
2324

2425
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -37,6 +38,8 @@ public class QueryId {
3738

3839
public static final QueryId MOCK_QUERY_ID = QueryId.valueOf("mock_query_id");
3940

41+
private static final int DATANODE_ID = IoTDBDescriptor.getInstance().getConfig().getDataNodeId();
42+
4043
private final String id;
4144

4245
private int nextPlanNodeIndex;
@@ -67,6 +70,10 @@ public String getId() {
6770
return id;
6871
}
6972

73+
public static int getDataNodeId() {
74+
return DATANODE_ID;
75+
}
76+
7077
@Override
7178
public String toString() {
7279
return id;

0 commit comments

Comments
 (0)