[client-v2, jdbc-v2] queryID auto-generation

client-v2/src/main/java/com/clickhouse/client/api/ClickHouseException.java

@@ -3,6 +3,8 @@
 public class ClickHouseException extends RuntimeException {
     protected boolean isRetryable  = false;
 
+    protected String queryId;
+
     public ClickHouseException(String message) {
         super(message);
     }
@@ -11,8 +13,21 @@ public ClickHouseException(String message, Throwable cause) {
         super(message, cause);
     }
 
+    public ClickHouseException(String message, Throwable cause, String queryId) {
+        super(message, cause);
+        this.queryId = queryId;
+    }
+
     public ClickHouseException(Throwable cause) {
         super(cause);
     }
     public boolean isRetryable() { return isRetryable; }
+
+    public void setQueryId(String queryId) {
+        this.queryId = queryId;
+    }
+
+    public String getQueryId() {
+        return queryId;
+    }
 }

client-v2/src/main/java/com/clickhouse/client/api/Client.java

@@ -137,21 +137,18 @@ public class Client implements AutoCloseable {
     // Server context
     private String dbUser;
     private String serverVersion;
-    private Object metricsRegistry;
-    private int retries;
+    private final Object metricsRegistry;
+    private final int retries;
     private LZ4Factory lz4Factory = null;
+    private final Supplier<String> queryIdGenerator;
 
     private Client(Collection<Endpoint> endpoints, Map<String,String> configuration,
-                   ExecutorService sharedOperationExecutor, ColumnToMethodMatchingStrategy columnToMethodMatchingStrategy) {
-        this(endpoints, configuration, sharedOperationExecutor, columnToMethodMatchingStrategy, null);
-    }
-
-    private Client(Collection<Endpoint> endpoints, Map<String,String> configuration,
-                   ExecutorService sharedOperationExecutor, ColumnToMethodMatchingStrategy columnToMethodMatchingStrategy, Object metricsRegistry) {
-        // Simple initialization
+                   ExecutorService sharedOperationExecutor, ColumnToMethodMatchingStrategy columnToMethodMatchingStrategy,
+                   Object metricsRegistry, Supplier<String> queryIdGenerator) {
         this.configuration = ClientConfigProperties.parseConfigMap(configuration);
         this.readOnlyConfig = Collections.unmodifiableMap(configuration);
         this.metricsRegistry = metricsRegistry;
+        this.queryIdGenerator = queryIdGenerator;
 
         // Serialization
         this.pojoSerDe = new POJOSerDe(columnToMethodMatchingStrategy);
@@ -266,6 +263,8 @@ public static class Builder {
         private ExecutorService sharedOperationExecutor = null;
         private ColumnToMethodMatchingStrategy columnToMethodMatchingStrategy;
         private Object metricRegistry = null;
+        private Supplier<String> queryIdGenerator;
+
         public Builder() {
             this.endpoints = new HashSet<>();
             this.configuration = new HashMap<>();
@@ -1085,6 +1084,16 @@ public Builder useHttpFormDataForQuery(boolean enable) {
             return this;
         }
 
+        /**
+         * Sets query id generator. Will be used when operation settings (InsertSettings, QuerySettings) do not have query id set.
+         * @param supplier
+         * @return
+         */
+        public Builder setQueryIdGenerator(Supplier<String> supplier) {
+            this.queryIdGenerator = supplier;
+            return this;
+        }
+
         public Client build() {
             // check if endpoint are empty. so can not initiate client
             if (this.endpoints.isEmpty()) {
@@ -1143,7 +1152,7 @@ public Client build() {
             }
 
             return new Client(this.endpoints, this.configuration, this.sharedOperationExecutor,
-                this.columnToMethodMatchingStrategy, this.metricRegistry);
+                this.columnToMethodMatchingStrategy, this.metricRegistry, this.queryIdGenerator);
         }
     }
 
@@ -1282,6 +1291,9 @@ public CompletableFuture<InsertResponse> insert(String tableName, List<?> data,
         final int maxRetries = retry == null ? 0 : retry;
 
         requestSettings.setOption(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), format);
+        if (requestSettings.getQueryId() == null && queryIdGenerator != null) {
+            requestSettings.setQueryId(queryIdGenerator.get());
+        }
         Supplier<InsertResponse> supplier = () -> {
             long startTime = System.nanoTime();
             // Selecting some node
@@ -1328,8 +1340,8 @@ public CompletableFuture<InsertResponse> insert(String tableName, List<?> data,
                     metrics.setQueryId(queryId);
                     return new InsertResponse(metrics);
                 } catch (Exception e) {
-                    lastException = httpClientHelper.wrapException(String.format("Query request failed (Attempt: %s/%s - Duration: %s)",
-                            (i + 1), (maxRetries + 1), durationSince(startTime)), e);
+                    String msg = requestExMsg("Insert", (i + 1), durationSince(startTime).toMillis(), requestSettings.getQueryId());
+                    lastException = httpClientHelper.wrapException(msg, e, requestSettings.getQueryId());
                     if (httpClientHelper.shouldRetry(e, requestSettings.getAllSettings())) {
                         LOG.warn("Retrying.", e);
                         selectedEndpoint = getNextAliveNode();
@@ -1338,8 +1350,10 @@ public CompletableFuture<InsertResponse> insert(String tableName, List<?> data,
                     }
                 }
             }
-            throw new ClientException("Insert request failed after attempts: " + (maxRetries + 1) + " - Duration: " + durationSince(startTime), lastException);
-        };
+
+            String errMsg = requestExMsg("Insert", retries, durationSince(startTime).toMillis(), requestSettings.getQueryId());
+            LOG.warn(errMsg);
+            throw (lastException == null ? new ClientException(errMsg) : lastException);        };
 
         return runAsyncOperation(supplier, requestSettings.getAllSettings());
 
@@ -1499,6 +1513,9 @@ public CompletableFuture<InsertResponse> insert(String tableName,
         }
         sqlStmt.append(" FORMAT ").append(format.name());
         requestSettings.serverSetting(ClickHouseHttpProto.QPARAM_QUERY_STMT, sqlStmt.toString());
+        if (requestSettings.getQueryId() == null && queryIdGenerator != null) {
+            requestSettings.setQueryId(queryIdGenerator.get());
+        }
         responseSupplier = () -> {
             long startTime = System.nanoTime();
             // Selecting some node
@@ -1530,8 +1547,8 @@ public CompletableFuture<InsertResponse> insert(String tableName,
                     metrics.setQueryId(queryId);
                     return new InsertResponse(metrics);
                 } catch (Exception e) {
-                    lastException = httpClientHelper.wrapException(String.format("Insert failed (Attempt: %s/%s - Duration: %s)",
-                            (i + 1), (retries + 1), durationSince(startTime)), e);
+                    String msg = requestExMsg("Insert", (i + 1), durationSince(startTime).toMillis(), requestSettings.getQueryId());
+                    lastException = httpClientHelper.wrapException(msg, e, requestSettings.getQueryId());
                     if (httpClientHelper.shouldRetry(e, requestSettings.getAllSettings())) {
                         LOG.warn("Retrying.", e);
                         selectedEndpoint = getNextAliveNode();
@@ -1548,8 +1565,9 @@ public CompletableFuture<InsertResponse> insert(String tableName,
                     }
                 }
             }
-            LOG.warn("Insert request failed after attempts: {} - Duration: {}", retries + 1, durationSince(startTime));
-            throw (lastException == null ? new ClientException("Failed to complete insert operation") : lastException);
+            String errMsg = requestExMsg("Insert", retries, durationSince(startTime).toMillis(), requestSettings.getQueryId());
+            LOG.warn(errMsg);
+            throw (lastException == null ? new ClientException(errMsg) : lastException);
         };
 
         return runAsyncOperation(responseSupplier, requestSettings.getAllSettings());
@@ -1598,7 +1616,7 @@ public CompletableFuture<QueryResponse> query(String sqlQuery, QuerySettings set
      *
      * <b>Notes:</b>
      * <ul>
-     * <li>Server response format can be specified thru {@code settings} or in SQL query.</li>
+     * <li>Server response format can be specified through {@code settings} or in SQL query.</li>
      * <li>If specified in both, the {@code sqlQuery} will take precedence.</li>
      * </ul>
      *
@@ -1623,6 +1641,10 @@ public CompletableFuture<QueryResponse> query(String sqlQuery, Map<String, Objec
             requestSettings.setOption(HttpAPIClientHelper.KEY_STATEMENT_PARAMS, queryParams);
         }
 
+        if (requestSettings.getQueryId() == null && queryIdGenerator != null) {
+            requestSettings.setQueryId(queryIdGenerator.get());
+        }
+
         Supplier<QueryResponse> responseSupplier = () -> {
                 long startTime = System.nanoTime();
                 // Selecting some node
@@ -1666,8 +1688,8 @@ public CompletableFuture<QueryResponse> query(String sqlQuery, Map<String, Objec
 
                     } catch (Exception e) {
                         HttpAPIClientHelper.closeQuietly(httpResponse);
-                        lastException = httpClientHelper.wrapException(String.format("Query request failed (Attempt: %s/%s - Duration: %s)",
-                                (i + 1), (retries + 1), durationSince(startTime)), e);
+                        String msg = requestExMsg("Query", (i + 1), durationSince(startTime).toMillis(), requestSettings.getQueryId());
+                        lastException = httpClientHelper.wrapException(msg, e, requestSettings.getQueryId());
                         if (httpClientHelper.shouldRetry(e, requestSettings.getAllSettings())) {
                             LOG.warn("Retrying.", e);
                             selectedEndpoint = getNextAliveNode();
@@ -1676,8 +1698,9 @@ public CompletableFuture<QueryResponse> query(String sqlQuery, Map<String, Objec
                         }
                     }
                 }
-                LOG.warn("Query request failed after attempts: {} - Duration: {}", retries + 1, durationSince(startTime));
-                throw (lastException == null ? new ClientException("Failed to complete query") : lastException);
+                String errMsg = requestExMsg("Query", retries, durationSince(startTime).toMillis(), requestSettings.getQueryId());
+                LOG.warn(errMsg);
+                throw (lastException == null ? new ClientException(errMsg) : lastException);
             };
 
         return runAsyncOperation(responseSupplier, requestSettings.getAllSettings());
@@ -2155,4 +2178,8 @@ private Map<String, Object> buildRequestSettings(Map<String, Object> opSettings)
     private Duration durationSince(long sinceNanos) {
         return Duration.ofNanos(System.nanoTime() - sinceNanos);
     }
+
+    private String requestExMsg(String operation, int attempt, long operationDuration, String queryId) {
+        return operation + " request failed (attempt: " + attempt +", duration: " + operationDuration + "ms, queryId: " + queryId + ")";
+    }
 }

client-v2/src/main/java/com/clickhouse/client/api/ServerException.java

@@ -12,15 +12,14 @@ public class ServerException extends ClickHouseException {
 
     private final int transportProtocolCode;
 
-    public ServerException(int code, String message) {
-        this(code, message, 500);
-    }
+    private final String queryId;
 
-    public ServerException(int code, String message, int transportProtocolCode) {
+    public ServerException(int code, String message, int transportProtocolCode, String queryId) {
         super(message);
         this.code = code;
         this.transportProtocolCode = transportProtocolCode;
         this.isRetryable = discoverIsRetryable(code, message, transportProtocolCode);
+        this.queryId = queryId;
     }
 
     /**
@@ -41,8 +40,17 @@ public int getTransportProtocolCode() {
         return transportProtocolCode;
     }
 
+    @Override
     public boolean isRetryable() {
-        return isRetryable;
+        return super.isRetryable();
+    }
+
+    /**
+     * Returns query ID that is returned by server in {@link com.clickhouse.client.api.http.ClickHouseHttpProto#HEADER_QUERY_ID}
+     * @return query id non-null string
+     */
+    public String getQueryId() {
+        return queryId;
     }
 
     private boolean discoverIsRetryable(int code, String message, int transportProtocolCode) {
@@ -68,4 +76,27 @@ private boolean discoverIsRetryable(int code, String message, int transportProto
         };
         return false;
     }
+
+    /**
+     * Not every server code is listed - only most common
+     */
+    public enum ErrorCodes {
+
+        UNKNOWN(0),
+        TABLE_NOT_FOUND(60),
+        DATABASE_NOT_FOUND(81),
+        UNKNOWN_SETTING(115),
+
+        ;
+
+        private int code;
+
+        ErrorCodes(int code) {
+            this.code = code;
+        }
+
+        public int getCode() {
+            return code;
+        }
+    }
 }

client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java

@@ -93,6 +93,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -341,13 +342,16 @@ public CloseableHttpClient createHttpClient(boolean initSslContext, Map<String,
 //    private static final String ERROR_CODE_PREFIX_PATTERN = "Code: %d. DB::Exception:";
     private static final String ERROR_CODE_PREFIX_PATTERN = "%d. DB::Exception:";
 
+
     /**
      * Reads status line and if error tries to parse response body to get server error message.
      *
      * @param httpResponse - HTTP response
      * @return exception object with server code
      */
     public Exception readError(ClassicHttpResponse httpResponse) {
+        final Header qIdHeader = httpResponse.getFirstHeader(ClickHouseHttpProto.HEADER_QUERY_ID);
+        final String queryId = qIdHeader == null ? "" : qIdHeader.getValue();
         int serverCode = getHeaderInt(httpResponse.getFirstHeader(ClickHouseHttpProto.HEADER_EXCEPTION_CODE), 0);
         InputStream body = null;
         try {
@@ -408,10 +412,11 @@ public Exception readError(ClassicHttpResponse httpResponse) {
             if (msg.trim().isEmpty()) {
                 msg = String.format(ERROR_CODE_PREFIX_PATTERN, serverCode) + " <Unreadable error message> (transport error: " + httpResponse.getCode() + ")";
             }
-            return new ServerException(serverCode, "Code: " + msg, httpResponse.getCode());
+            return new ServerException(serverCode, "Code: " + msg + " (queryId= " + queryId + ")", httpResponse.getCode(), queryId);
         } catch (Exception e) {
             LOG.error("Failed to read error message", e);
-            return new ServerException(serverCode, String.format(ERROR_CODE_PREFIX_PATTERN, serverCode) + " <Unreadable error message> (transport error: " + httpResponse.getCode() + ")", httpResponse.getCode());
+            String msg = String.format(ERROR_CODE_PREFIX_PATTERN, serverCode) + " <Unreadable error message> (transport error: " + httpResponse.getCode() + ")";
+            return new ServerException(serverCode, msg + " (queryId= " + queryId + ")", httpResponse.getCode(), queryId);
         }
     }
 
@@ -799,7 +804,7 @@ public boolean shouldRetry(Throwable ex, Map<String, Object> requestSettings) {
 
     // This method wraps some client specific exceptions into specific ClientException or just ClientException
     // ClientException will be also wrapped
-    public RuntimeException wrapException(String message, Exception cause) {
+    public RuntimeException wrapException(String message, Exception cause, String queryId) {
         if (cause instanceof ClientException || cause instanceof ServerException) {
             return (RuntimeException) cause;
         }
@@ -810,14 +815,18 @@ public RuntimeException wrapException(String message, Exception cause) {
                 cause instanceof ConnectException ||
                 cause instanceof UnknownHostException ||
                 cause instanceof NoRouteToHostException) {
-            return new ConnectionInitiationException(message, cause);
+            ConnectionInitiationException ex = new ConnectionInitiationException(message, cause);
+            ex.setQueryId(queryId);
+            return ex;
         }
 
         if (cause instanceof SocketTimeoutException || cause instanceof IOException) {
-            return new DataTransferException(message, cause);
+            DataTransferException ex =  new DataTransferException(message, cause);
+            ex.setQueryId(queryId);
+            return ex;
         }
         // if we can not identify the exception explicitly we catch as our base exception ClickHouseException
-        return new ClickHouseException(message, cause);
+        return new ClickHouseException(message, cause, queryId);
     }
 
     private void correctUserAgentHeader(HttpRequest request, Map<String, Object> requestConfig) {

client-v2/src/main/java/com/clickhouse/client/api/query/Records.java

@@ -125,6 +125,14 @@ public long getResultRows() {
         return response.getMetrics().getMetric(ServerMetrics.RESULT_ROWS).getLong();
     }
 
+    /**
+     * Returns response query id
+     * @return query id of the request
+     */
+    public String getQueryId() {
+        return response.getQueryId();
+    }
+
     @Override
     public void close() throws Exception {
         response.close();