Skip to content

Commit 653083b

Browse files
authored
[feature](jdbc) support mapping varbinary type in JBDC catalog (#58215)
### What problem does this PR solve? Problem Summary: support varbinary type mapping in DB2,MYSQL,Oracle,PostgreSQL,SQLServer JDBC catalog. u can control this when create catalog with property "enable.mapping.varbinary", default value is false. if it's true, will mapping the binary type to doris varbinary type, if it's false, will mapping the binary type to doris string type. Followup #57821 ### Release note support mapping varbinary type in JBDC catalog
1 parent f4fdf54 commit 653083b

37 files changed

+242
-39
lines changed

docker/thirdparties/docker-compose/oracle/init/03-create-table.sql

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,3 +258,8 @@ create table doris_test.extreme_test_multi_block (
258258
t7 interval day(3) to second(6)
259259
);
260260

261+
CREATE TABLE doris_test.varbinary_test(
262+
"id" NUMBER(5,0),
263+
"NAME" VARCHAR2(20),
264+
"BLOB_COL" BLOB
265+
);

docker/thirdparties/docker-compose/oracle/init/04-insert.sql

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,4 +155,8 @@ SELECT
155155
t1, t2, t3, t4, t5, t6, t7
156156
FROM doris_test.extreme_test;
157157

158+
INSERT INTO doris_test.varbinary_test VALUES (1, 'empty', EMPTY_BLOB());
159+
INSERT INTO doris_test.varbinary_test VALUES (2, 'NULL', NULL);
160+
INSERT INTO doris_test.varbinary_test VALUES (3, 'normal', HEXTORAW('48656C6C6F20576F726C64'));
161+
158162
commit;

fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/DB2JdbcExecutor.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ protected Object getColumnValue(int columnIndex, ColumnType type, String[] repla
8181
case VARCHAR:
8282
case STRING:
8383
return resultSet.getObject(columnIndex + 1, String.class);
84+
case VARBINARY:
85+
return resultSet.getObject(columnIndex + 1, byte[].class);
8486
default:
8587
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
8688
}

fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,8 @@ private Object newGetColumnValue(int columnIndex, ColumnType type, String[] repl
101101
case VARCHAR:
102102
case STRING:
103103
return resultSet.getObject(columnIndex + 1);
104+
case VARBINARY:
105+
return resultSet.getObject(columnIndex + 1, byte[].class);
104106
default:
105107
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
106108
}
@@ -142,6 +144,9 @@ private Object oldGetColumnValue(int columnIndex, ColumnType type, String[] repl
142144
case STRING:
143145
Object stringVal = resultSet.getObject(columnIndex + 1);
144146
return resultSet.wasNull() ? null : stringVal;
147+
case VARBINARY:
148+
byte[] bytesVal = resultSet.getBytes(columnIndex + 1);
149+
return resultSet.wasNull() ? null : bytesVal;
145150
default:
146151
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
147152
}

fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/PostgreSQLJdbcExecutor.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,8 @@ protected Object getColumnValue(int columnIndex, ColumnType type, String[] repla
8787
case VARCHAR:
8888
case STRING:
8989
return resultSet.getObject(columnIndex + 1);
90+
case VARBINARY:
91+
return resultSet.getBytes(columnIndex + 1);
9092
case ARRAY:
9193
java.sql.Array array = resultSet.getArray(columnIndex + 1);
9294
return array == null ? null : convertArrayToList(array.getArray());

fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/SQLServerJdbcExecutor.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,8 @@ protected Object getColumnValue(int columnIndex, ColumnType type, String[] repla
8787
case VARCHAR:
8888
case STRING:
8989
return resultSet.getObject(columnIndex + 1);
90+
case VARBINARY:
91+
return resultSet.getObject(columnIndex + 1, byte[].class);
9092
default:
9193
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
9294
}

fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/TrinoJdbcExecutor.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,8 @@ protected Object getColumnValue(int columnIndex, ColumnType type, String[] repla
9898
}
9999
return Arrays.asList(dataArray);
100100
}
101+
case VARBINARY:
102+
return resultSet.getObject(columnIndex + 1, byte[].class);
101103
default:
102104
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
103105
}

fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.doris.common.plugin.CloudPluginDownloader.PluginType;
2828
import org.apache.doris.common.proc.BaseProcResult;
2929
import org.apache.doris.common.util.Util;
30+
import org.apache.doris.datasource.CatalogProperty;
3031
import org.apache.doris.datasource.ExternalCatalog;
3132

3233
import com.google.common.base.Preconditions;
@@ -134,7 +135,8 @@ public class JdbcResource extends Resource {
134135
CONNECTION_POOL_MAX_WAIT_TIME,
135136
CONNECTION_POOL_KEEP_ALIVE,
136137
TEST_CONNECTION,
137-
ExternalCatalog.USE_META_CACHE
138+
ExternalCatalog.USE_META_CACHE,
139+
CatalogProperty.ENABLE_MAPPING_VARBINARY
138140
).build();
139141

140142
// The default value of optional properties
@@ -155,6 +157,7 @@ public class JdbcResource extends Resource {
155157
OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(TEST_CONNECTION, "true");
156158
OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ExternalCatalog.USE_META_CACHE,
157159
String.valueOf(ExternalCatalog.DEFAULT_USE_META_CACHE));
160+
OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CatalogProperty.ENABLE_MAPPING_VARBINARY, "false");
158161
}
159162

160163
// timeout for both connection and read. 10 seconds is long enough.

fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,8 @@ private JdbcClient createJdbcClient() {
251251
.setConnectionPoolMaxSize(getConnectionPoolMaxSize())
252252
.setConnectionPoolMaxLifeTime(getConnectionPoolMaxLifeTime())
253253
.setConnectionPoolMaxWaitTime(getConnectionPoolMaxWaitTime())
254-
.setConnectionPoolKeepAlive(isConnectionPoolKeepAlive());
254+
.setConnectionPoolKeepAlive(isConnectionPoolKeepAlive())
255+
.setEnableMappingVarbinary(getEnableMappingVarbinary());
255256

256257
return JdbcClient.createJdbcClient(jdbcClientConfig);
257258
}

fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ public abstract class JdbcClient {
6767
protected boolean isOnlySpecifiedDatabase;
6868
protected Map<String, Boolean> includeDatabaseMap;
6969
protected Map<String, Boolean> excludeDatabaseMap;
70+
protected boolean enableMappingVarbinary;
7071

7172
public static JdbcClient createJdbcClient(JdbcClientConfig jdbcClientConfig) {
7273
String dbType = parseDbType(jdbcClientConfig.getJdbcUrl());
@@ -111,6 +112,7 @@ protected JdbcClient(JdbcClientConfig jdbcClientConfig) {
111112
this.dbType = parseDbType(jdbcUrl);
112113
initializeClassLoader(jdbcClientConfig);
113114
initializeDataSource(jdbcClientConfig);
115+
this.enableMappingVarbinary = jdbcClientConfig.isEnableMappingVarbinary();
114116
}
115117

116118
protected void setJdbcDriverSystemProperties() {

0 commit comments

Comments
 (0)