collect min max, also collect count

Author: yujun777

fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java

@@ -53,6 +53,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.BiFunction;
 import java.util.stream.Collectors;
 
 /**
@@ -62,7 +63,8 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
 
     private static final String BASIC_STATS_TEMPLATE = "SELECT "
             + "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) as min, "
-            + "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) as max "
+            + "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) as max, "
+            + "COUNT(1) as row_count "
             + "FROM `${dbName}`.`${tblName}` ${index}";
 
     private boolean keyColumnSampleTooManyRows = false;
@@ -109,19 +111,30 @@ protected void doSample() {
             LOG.debug("Will do sample collection for column {}", col.getName());
         }
         // Get basic stats, including min and max.
-        ResultRow minMax = collectMinMax();
-        String min = StatisticsUtil.escapeSQL(minMax != null && minMax.getValues().size() > 0
-                ? minMax.get(0) : null);
-        String max = StatisticsUtil.escapeSQL(minMax != null && minMax.getValues().size() > 1
-                ? minMax.get(1) : null);
+        ResultRow minMaxCount = collectMinMaxCount();
+        BiFunction<ResultRow, Integer, String> escapeCell = (row, i) ->
+                StatisticsUtil.escapeSQL(row != null && row.getValues().size() > i ? row.getValues().get(i) : null);
+        String min = escapeCell.apply(minMaxCount, 0);
+        String max = escapeCell.apply(minMaxCount, 1);
+        String tableRowCountStr = escapeCell.apply(minMaxCount, 2);
+        long tableRowCount = -1;
+        if (tableRowCountStr != null) {
+            try {
+                tableRowCount = Long.parseLong(tableRowCountStr);
+            } catch (Exception e) {
+                // ignore parse exception
+            }
+        }
+        if (tableRowCount < 0) {
+            tableRowCount = info.indexId == -1
+                    ? tbl.getRowCount()
+                    : ((OlapTable) tbl).getRowCountForIndex(info.indexId, false);
+        }
 
         Map<String, String> params = buildSqlParams();
         params.put("min", StatisticsUtil.quote(min));
         params.put("max", StatisticsUtil.quote(max));
         params.put("hotValueCollectCount", String.valueOf(SessionVariable.getHotValueCollectCount()));
-        long tableRowCount = info.indexId == -1
-                ? tbl.getRowCount()
-                : ((OlapTable) tbl).getRowCountForIndex(info.indexId, false);
         getSampleParams(params, tableRowCount);
         StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
         String sql;
@@ -136,7 +149,7 @@ protected void doSample() {
         runQuery(sql);
     }
 
-    protected ResultRow collectMinMax() {
+    protected ResultRow collectMinMaxCount() {
         long startTime = System.currentTimeMillis();
         Map<String, String> params = buildSqlParams();
         StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
@@ -146,7 +159,7 @@ protected ResultRow collectMinMax() {
             stmtExecutor = new StmtExecutor(r.connectContext, sql);
             resultRow = stmtExecutor.executeInternalQuery().get(0);
             if (LOG.isDebugEnabled()) {
-                LOG.debug("Cost time in millisec: " + (System.currentTimeMillis() - startTime) + " Min max SQL: "
+                LOG.debug("Cost time in millisec: " + (System.currentTimeMillis() - startTime) + " Min max count SQL: "
                         + sql + " QueryId: " + DebugUtil.printId(stmtExecutor.getContext().queryId()));
             }
             // Release the reference to stmtExecutor, reduce memory usage.

fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java

@@ -111,8 +111,8 @@ public void testKeyColumnUseLimitAndNot(@Mocked CatalogIf catalogIf, @Mocked Dat
 
         new Expectations() {
             {
-                tableIf.getRowCount();
-                result = 20000000;
+                // tableIf.getRowCount();
+                // result = 20000000;
                 tableIf.getId();
                 result = 30001;
                 catalogIf.getId();
@@ -126,10 +126,11 @@ public void testKeyColumnUseLimitAndNot(@Mocked CatalogIf catalogIf, @Mocked Dat
 
         new MockUp<OlapAnalysisTask>() {
             @Mock
-            public ResultRow collectMinMax() {
+            public ResultRow collectMinMaxCount() {
                 List<String> values = Lists.newArrayList();
                 values.add("1");
                 values.add("2");
+                values.add("20000000");
                 return new ResultRow(values);
             }