Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -258,3 +258,8 @@ create table doris_test.extreme_test_multi_block (
t7 interval day(3) to second(6)
);

CREATE TABLE doris_test.varbinary_test(
"id" NUMBER(5,0),
"NAME" VARCHAR2(20),
"BLOB_COL" BLOB
);
4 changes: 4 additions & 0 deletions docker/thirdparties/docker-compose/oracle/init/04-insert.sql
Original file line number Diff line number Diff line change
Expand Up @@ -155,4 +155,8 @@ SELECT
t1, t2, t3, t4, t5, t6, t7
FROM doris_test.extreme_test;

INSERT INTO doris_test.varbinary_test VALUES (1, 'empty', EMPTY_BLOB());
INSERT INTO doris_test.varbinary_test VALUES (2, 'NULL', NULL);
INSERT INTO doris_test.varbinary_test VALUES (3, 'normal', HEXTORAW('48656C6C6F20576F726C64'));

commit;
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ protected Object getColumnValue(int columnIndex, ColumnType type, String[] repla
case VARCHAR:
case STRING:
return resultSet.getObject(columnIndex + 1, String.class);
case VARBINARY:
return resultSet.getObject(columnIndex + 1, byte[].class);
default:
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,8 @@ private Object newGetColumnValue(int columnIndex, ColumnType type, String[] repl
case VARCHAR:
case STRING:
return resultSet.getObject(columnIndex + 1);
case VARBINARY:
return resultSet.getObject(columnIndex + 1, byte[].class);
default:
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
}
Expand Down Expand Up @@ -142,6 +144,9 @@ private Object oldGetColumnValue(int columnIndex, ColumnType type, String[] repl
case STRING:
Object stringVal = resultSet.getObject(columnIndex + 1);
return resultSet.wasNull() ? null : stringVal;
case VARBINARY:
byte[] bytesVal = resultSet.getBytes(columnIndex + 1);
return resultSet.wasNull() ? null : bytesVal;
default:
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ protected Object getColumnValue(int columnIndex, ColumnType type, String[] repla
case VARCHAR:
case STRING:
return resultSet.getObject(columnIndex + 1);
case VARBINARY:
return resultSet.getBytes(columnIndex + 1);
case ARRAY:
java.sql.Array array = resultSet.getArray(columnIndex + 1);
return array == null ? null : convertArrayToList(array.getArray());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ protected Object getColumnValue(int columnIndex, ColumnType type, String[] repla
case VARCHAR:
case STRING:
return resultSet.getObject(columnIndex + 1);
case VARBINARY:
return resultSet.getObject(columnIndex + 1, byte[].class);
default:
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,8 @@ protected Object getColumnValue(int columnIndex, ColumnType type, String[] repla
}
return Arrays.asList(dataArray);
}
case VARBINARY:
return resultSet.getObject(columnIndex + 1, byte[].class);
default:
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.doris.common.plugin.CloudPluginDownloader.PluginType;
import org.apache.doris.common.proc.BaseProcResult;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CatalogProperty;
import org.apache.doris.datasource.ExternalCatalog;

import com.google.common.base.Preconditions;
Expand Down Expand Up @@ -134,7 +135,8 @@ public class JdbcResource extends Resource {
CONNECTION_POOL_MAX_WAIT_TIME,
CONNECTION_POOL_KEEP_ALIVE,
TEST_CONNECTION,
ExternalCatalog.USE_META_CACHE
ExternalCatalog.USE_META_CACHE,
CatalogProperty.ENABLE_MAPPING_VARBINARY
).build();

// The default value of optional properties
Expand All @@ -155,6 +157,7 @@ public class JdbcResource extends Resource {
OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(TEST_CONNECTION, "true");
OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ExternalCatalog.USE_META_CACHE,
String.valueOf(ExternalCatalog.DEFAULT_USE_META_CACHE));
OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CatalogProperty.ENABLE_MAPPING_VARBINARY, "false");
}

// timeout for both connection and read. 10 seconds is long enough.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,8 @@ private JdbcClient createJdbcClient() {
.setConnectionPoolMaxSize(getConnectionPoolMaxSize())
.setConnectionPoolMaxLifeTime(getConnectionPoolMaxLifeTime())
.setConnectionPoolMaxWaitTime(getConnectionPoolMaxWaitTime())
.setConnectionPoolKeepAlive(isConnectionPoolKeepAlive());
.setConnectionPoolKeepAlive(isConnectionPoolKeepAlive())
.setEnableMappingVarbinary(getEnableMappingVarbinary());

return JdbcClient.createJdbcClient(jdbcClientConfig);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ public abstract class JdbcClient {
protected boolean isOnlySpecifiedDatabase;
protected Map<String, Boolean> includeDatabaseMap;
protected Map<String, Boolean> excludeDatabaseMap;
protected boolean enableMappingVarbinary;

public static JdbcClient createJdbcClient(JdbcClientConfig jdbcClientConfig) {
String dbType = parseDbType(jdbcClientConfig.getJdbcUrl());
Expand Down Expand Up @@ -111,6 +112,7 @@ protected JdbcClient(JdbcClientConfig jdbcClientConfig) {
this.dbType = parseDbType(jdbcUrl);
initializeClassLoader(jdbcClientConfig);
initializeDataSource(jdbcClientConfig);
this.enableMappingVarbinary = jdbcClientConfig.isEnableMappingVarbinary();
}

protected void setJdbcDriverSystemProperties() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.doris.datasource.jdbc.client;

import org.apache.doris.catalog.JdbcResource;
import org.apache.doris.datasource.CatalogProperty;

import com.google.common.collect.Maps;

Expand All @@ -39,6 +40,9 @@ public class JdbcClientConfig implements Cloneable {
private int connectionPoolMaxWaitTime;
private int connectionPoolMaxLifeTime;
private boolean connectionPoolKeepAlive;
// Whether to enable mapping BINARY to doris VARBINARY
// default: false, mapping to doris string type
private boolean enableMappingVarbinary;

private Map<String, Boolean> includeDatabaseMap;
private Map<String, Boolean> excludeDatabaseMap;
Expand All @@ -61,6 +65,8 @@ public JdbcClientConfig() {
this.includeDatabaseMap = Maps.newHashMap();
this.excludeDatabaseMap = Maps.newHashMap();
this.customizedProperties = Maps.newHashMap();
this.enableMappingVarbinary = Boolean.parseBoolean(
JdbcResource.getDefaultPropertyValue(CatalogProperty.ENABLE_MAPPING_VARBINARY));
}

@Override
Expand Down Expand Up @@ -226,6 +232,15 @@ public JdbcClientConfig setExcludeDatabaseMap(Map<String, Boolean> excludeDataba
return this;
}

public JdbcClientConfig setEnableMappingVarbinary(boolean enableMappingVarbinary) {
this.enableMappingVarbinary = enableMappingVarbinary;
return this;
}

public boolean isEnableMappingVarbinary() {
return enableMappingVarbinary;
}

public void setCustomizedProperties(Map<String, String> customizedProperties) {
this.customizedProperties = customizedProperties;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,9 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
case "LONG VARGRAPHIC":
case "XML":
return ScalarType.createStringType();
case "BLOB":
return enableMappingVarbinary ? ScalarType.createVarbinaryType(fieldSchema.requiredColumnSize())
: ScalarType.createStringType();
default:
return Type.UNSUPPORTED;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -283,9 +283,14 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
return ScalarType.createCharType(fieldSchema.requiredColumnSize());
case "VARCHAR":
return ScalarType.createVarcharType(fieldSchema.requiredColumnSize());
case "TINYBLOB":
case "BLOB":
case "MEDIUMBLOB":
case "LONGBLOB":
case "BINARY":
case "VARBINARY":
return ScalarType.createVarbinaryType(fieldSchema.requiredColumnSize());
return enableMappingVarbinary ? ScalarType.createVarbinaryType(fieldSchema.requiredColumnSize())
: ScalarType.createStringType();
case "BIT":
if (fieldSchema.requiredColumnSize() == 1) {
return Type.BOOLEAN;
Expand All @@ -298,10 +303,6 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
case "TEXT":
case "MEDIUMTEXT":
case "LONGTEXT":
case "TINYBLOB":
case "BLOB":
case "MEDIUMBLOB":
case "LONGBLOB":
case "STRING":
case "SET":
case "ENUM":
Expand Down Expand Up @@ -421,6 +422,8 @@ private Type dorisTypeToDoris(JdbcFieldSchema fieldSchema) {
return ScalarType.createHllType();
case "BITMAP":
return Type.BITMAP;
case "VARBINARY":
return ScalarType.createVarbinaryType(fieldSchema.requiredColumnSize());
default:
return Type.UNSUPPORTED;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,8 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
case "CLOB":
return ScalarType.createStringType();
case "BLOB":
return enableMappingVarbinary ? ScalarType.createVarbinaryType(fieldSchema.requiredColumnSize())
: ScalarType.createStringType();
case "NCLOB":
case "BFILE":
case "BINARY_FLOAT":
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -165,10 +165,12 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
case "macaddr":
case "varbit":
case "uuid":
case "bytea":
case "json":
case "jsonb":
return ScalarType.createStringType();
case "bytea": // https://www.postgresql.org/docs/12/datatype-binary.html#DATATYPE-BINARY-TABLE
return enableMappingVarbinary ? ScalarType.createVarbinaryType(fieldSchema.requiredColumnSize())
: ScalarType.createStringType();
default: {
if (fieldSchema.getDataType() == Types.ARRAY && pgType.startsWith("_")) {
return convertArrayType(fieldSchema);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
case "image":
case "binary":
case "varbinary":
return enableMappingVarbinary ? ScalarType.createVarbinaryType(fieldSchema.requiredColumnSize())
: ScalarType.createStringType();
default:
return Type.UNSUPPORTED;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,3 +70,15 @@ mysql
-- !select_xml_tvf --
1000 <catalog><book><author> Gambardella Matthew</author><title>XML Developers Guide</title><genre>Computer</genre><price>44.95</price><publish_date>2000-10-01</publish_date><description>An in-depth look at creating application\n with XML</description></book></catalog>

-- !varbinary_desc --
BLOB_COLUMN varbinary(1048576) Yes true \N
ID_COLUMN int No true \N

-- !varbinary_select --
1 0x48656C6C6F20576F726C6421

-- !varbinary_select_after_insert --
1 0x48656C6C6F20576F726C6421
2 0x48656C6C6F20576F726C6421
3 \N

Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ bigint bigint Yes true \N NONE
bigint_u largeint Yes true \N NONE
binary varbinary(12) Yes true \N NONE
bit text Yes true \N NONE
blob text Yes true \N NONE
blob varbinary(65535) Yes true \N NONE
boolean tinyint Yes true \N NONE
char char(5) Yes true \N NONE
date date Yes true \N NONE
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2310,3 +2310,10 @@ mysql
123 abc
234 bcd

-- !varbinary_test --
1 0x01030000000100000005000000000000000000000000000000000000000000000000000000000000000000F03F000000000000F03F000000000000F03F000000000000F03F000000000000000000000000000000000000000000000000

-- !varbinary_test_after_insert --
1 0x01030000000100000005000000000000000000000000000000000000000000000000000000000000000000F03F000000000000F03F000000000000F03F000000000000F03F000000000000000000000000000000000000000000000000
3 0x0101000000000000000000F03F0000000000000040

Original file line number Diff line number Diff line change
Expand Up @@ -149,3 +149,20 @@ information_schema
mysql
sys

-- !desc --
binary_value varbinary(20) Yes true \N
bit_value boolean No true \N
id int No true \N
varbinary_value varbinary(20) Yes true \N

-- !query --
1 false 0x4D616B6520446F72697320477265617421000000 0x4D616B6520446F72697320477265617421
2 true 0x4D616B6520446F72697320477265617421000000 0x4D616B6520446F72697320477265617421
3 true 0x4D616B6520446F72697320477265617421000000 0x4D616B6520446F72697320477265617421

-- !query_after_insert --
1 false 0x4D616B6520446F72697320477265617421000000 0x4D616B6520446F72697320477265617421
2 true 0x4D616B6520446F72697320477265617421000000 0x4D616B6520446F72697320477265617421
3 true 0x4D616B6520446F72697320477265617421000000 0x4D616B6520446F72697320477265617421
4 true 0xABAB000000000000000000000000000000000000 0xAB

Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !select_all_types_nullable --
\N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N
0 0 0 0 0 0 0.00 0.00000 0E-10 0E-30 0 0.0 0 0 0 0 0 0 0 0.0 0 0.00 0.00000 0E-10 0E-30 1901 00:00:00 00:00:00 00:00:00 1000-01-01 1000-01-01T00:00 1970-01-01T00:00:01 1970-01-01T00:00:01 1970-01-01T00:00:01 0x {} 0x00 Value1
0 0 0 0 0 0 0.00 0.00000 0E-10 0E-30 0 0.0 0 -128 -32768 -8388608 -2147483648 -9223372036854775808 -1.797693134862316e+308 -3.40282E38 -9999999999 -9999999.99 -9999999999999.99999 -9999999999999999999999999999.9999999999 -99999999999999999999999999999999999.999999999999999999999999999999 1901 -838:59:59 -838:59:59 -838:59:59 1000-01-01 1000-01-01T00:00 1970-01-01T00:00:01 1970-01-01T00:00:01 1970-01-01T00:00:01 0x {} 0x00 Value1
255 65535 16777215 4294967295 18446744073709551615 9999999999 9999999.99 9999999999999.99999 9999999999999999999999999999.9999999999 99999999999999999999999999999999999.999999999999999999999999999999 1.797693134862316e+308 3.40282E38 1 127 32767 8388607 2147483647 9223372036854775807 1.797693134862316e+308 3.40282E38 9999999999 9999999.99 9999999999999.99999 9999999999999999999999999999.9999999999 99999999999999999999999999999999999.999999999999999999999999999999 2155 838:59:59 838:59:59 838:59:59 9999-12-31 9999-12-31T23:59:59 2038-01-19T03:14:07 2038-01-19T03:14:07.499 2038-01-19T03:14:07.499999 zzzz abcdefgh max 0x6D6178424C4F42 {"key": "maxJSON"} Option1,Option2,Option3 0x3F Value3
\N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N
0 0 0 0 0 0 0.00 0.00000 0E-10 0E-30 0 0.0 0 0 0 0 0 0 0 0.0 0 0.00 0.00000 0E-10 0E-30 1901 00:00:00 00:00:00 00:00:00 1000-01-01 1000-01-01T00:00 1970-01-01T00:00:01 1970-01-01T00:00:01 1970-01-01T00:00:01 {} 0x00 Value1
0 0 0 0 0 0 0.00 0.00000 0E-10 0E-30 0 0.0 0 -128 -32768 -8388608 -2147483648 -9223372036854775808 -1.797693134862316e+308 -3.40282E38 -9999999999 -9999999.99 -9999999999999.99999 -9999999999999999999999999999.9999999999 -99999999999999999999999999999999999.999999999999999999999999999999 1901 -838:59:59 -838:59:59 -838:59:59 1000-01-01 1000-01-01T00:00 1970-01-01T00:00:01 1970-01-01T00:00:01 1970-01-01T00:00:01 {} 0x00 Value1
255 65535 16777215 4294967295 18446744073709551615 9999999999 9999999.99 9999999999999.99999 9999999999999999999999999999.9999999999 99999999999999999999999999999999999.999999999999999999999999999999 1.797693134862316e+308 3.40282E38 1 127 32767 8388607 2147483647 9223372036854775807 1.797693134862316e+308 3.40282E38 9999999999 9999999.99 9999999999999.99999 9999999999999999999999999999.9999999999 99999999999999999999999999999999999.999999999999999999999999999999 2155 838:59:59 838:59:59 838:59:59 9999-12-31 9999-12-31T23:59:59 2038-01-19T03:14:07 2038-01-19T03:14:07.499 2038-01-19T03:14:07.499999 zzzz abcdefgh max {"key": "maxJSON"} Option1,Option2,Option3 0x3F Value3

-- !select_all_types_non_nullable --
0 0 0 0 0 0 0.00 0.00000 0E-10 0E-30 0 0.0 0 0 0 0 0 0 0 0.0 0 0.00 0.00000 0E-10 0E-30 1901 00:00:00 00:00:00 00:00:00 1000-01-01 1000-01-01T00:00 1970-01-01T00:00:01 1970-01-01T00:00:01 1970-01-01T00:00:01 0x {} 0x00 Value1
0 0 0 0 0 0 0.00 0.00000 0E-10 0E-30 0 0.0 0 -128 -32768 -8388608 -2147483648 -9223372036854775808 -1.797693134862316e+308 -3.40282E38 -9999999999 -9999999.99 -9999999999999.99999 -9999999999999999999999999999.9999999999 -99999999999999999999999999999999999.999999999999999999999999999999 1901 -838:59:59 -838:59:59 -838:59:59 1000-01-01 1000-01-01T00:00 1970-01-01T00:00:01 1970-01-01T00:00:01 1970-01-01T00:00:01 0x {} 0x00 Value1
255 65535 16777215 4294967295 18446744073709551615 9999999999 9999999.99 9999999999999.99999 9999999999999999999999999999.9999999999 99999999999999999999999999999999999.999999999999999999999999999999 1.797693134862316e+308 3.40282E38 1 127 32767 8388607 2147483647 9223372036854775807 1.797693134862316e+308 3.40282E38 9999999999 9999999.99 9999999999999.99999 9999999999999999999999999999.9999999999 99999999999999999999999999999999999.999999999999999999999999999999 2155 838:59:59 838:59:59 838:59:59 9999-12-31 9999-12-31T23:59:59 2038-01-19T03:14:07 2038-01-19T03:14:07.499 2038-01-19T03:14:07.499999 zzzz abcdefgh max 0x6D6178424C4F42 {"key": "maxJSON"} Option1,Option2,Option3 0x3F Value3
0 0 0 0 0 0 0.00 0.00000 0E-10 0E-30 0 0.0 0 0 0 0 0 0 0 0.0 0 0.00 0.00000 0E-10 0E-30 1901 00:00:00 00:00:00 00:00:00 1000-01-01 1000-01-01T00:00 1970-01-01T00:00:01 1970-01-01T00:00:01 1970-01-01T00:00:01 {} 0x00 Value1
0 0 0 0 0 0 0.00 0.00000 0E-10 0E-30 0 0.0 0 -128 -32768 -8388608 -2147483648 -9223372036854775808 -1.797693134862316e+308 -3.40282E38 -9999999999 -9999999.99 -9999999999999.99999 -9999999999999999999999999999.9999999999 -99999999999999999999999999999999999.999999999999999999999999999999 1901 -838:59:59 -838:59:59 -838:59:59 1000-01-01 1000-01-01T00:00 1970-01-01T00:00:01 1970-01-01T00:00:01 1970-01-01T00:00:01 {} 0x00 Value1
255 65535 16777215 4294967295 18446744073709551615 9999999999 9999999.99 9999999999999.99999 9999999999999999999999999999.9999999999 99999999999999999999999999999999999.999999999999999999999999999999 1.797693134862316e+308 3.40282E38 1 127 32767 8388607 2147483647 9223372036854775807 1.797693134862316e+308 3.40282E38 9999999999 9999999.99 9999999999999.99999 9999999999999999999999999999.9999999999 99999999999999999999999999999999999.999999999999999999999999999999 2155 838:59:59 838:59:59 838:59:59 9999-12-31 9999-12-31T23:59:59 2038-01-19T03:14:07 2038-01-19T03:14:07.499 2038-01-19T03:14:07.499999 zzzz abcdefgh max {"key": "maxJSON"} Option1,Option2,Option3 0x3F Value3

-- !select_varchar --
a
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ timestamp3 datetime(6) Yes true \N
char char(5) Yes true \N
varchar varchar(10) Yes true \N
text text Yes true \N
blob text Yes true \N
blob varbinary(65535) Yes true \N
json text Yes true \N
set text Yes true \N
bit text Yes true \N
Expand Down Expand Up @@ -89,7 +89,7 @@ timestamp3 datetime(6) No true \N
char char(5) No true \N
varchar varchar(10) No true \N
text text No true \N
blob text No true \N
blob varbinary(65535) No true \N
json text No true \N
set text No true \N
bit text No true \N
Expand Down
Loading
Loading