Skip to content

Commit ec0bdc7

Browse files
CalvinKirsCalvinKirs
authored
[fix](paimon-catalog)Fix OSS access when using DLS endpoint (#58099)
### What This PR Does - Adds explicit differentiation between **OSSHDFS** and **OSS** storage types. - Ensures DLS endpoints automatically route through the OSSHDFS protocol. - Avoids incorrectly treating DLS endpoints as standard OSS, which previously caused runtime errors. ### Why This Matters This update ensures stable compatibility with DLS endpoints and prevents unexpected failures for users whose OSS configuration points to DLS.
1 parent fe7938d commit ec0bdc7

File tree

6 files changed

+95
-48
lines changed

6 files changed

+95
-48
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,9 @@ public TFileType getTFileTypeForBE() {
312312
.equals(schema)) && AzurePropertyUtils.isOneLakeLocation(normalizedLocation)) {
313313
return TFileType.FILE_HDFS;
314314
}
315+
if (StringUtils.isNotBlank(normalizedLocation) && isHdfsOnOssEndpoint(normalizedLocation)) {
316+
return TFileType.FILE_HDFS;
317+
}
315318
return SchemaTypeMapper.fromSchemaToFileType(schema);
316319
}
317320

‎fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java‎

Lines changed: 9 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -49,22 +49,22 @@
4949
public class OSSHdfsProperties extends HdfsCompatibleProperties {
5050

5151
@Setter
52-
@ConnectorProperty(names = {"oss.hdfs.endpoint",
53-
"dlf.endpoint", "dlf.catalog.endpoint", "oss.endpoint"},
52+
@ConnectorProperty(names = {"oss.hdfs.endpoint", "oss.endpoint",
53+
"dlf.endpoint", "dlf.catalog.endpoint"},
5454
description = "The endpoint of OSS.")
5555
protected String endpoint = "";
5656

57-
@ConnectorProperty(names = {"oss.hdfs.access_key", "dlf.access_key", "dlf.catalog.accessKeyId", "oss.access_key"},
57+
@ConnectorProperty(names = {"oss.hdfs.access_key", "oss.access_key", "dlf.access_key", "dlf.catalog.accessKeyId"},
5858
sensitive = true,
5959
description = "The access key of OSS.")
6060
protected String accessKey = "";
6161

62-
@ConnectorProperty(names = {"oss.hdfs.secret_key", "dlf.secret_key", "dlf.catalog.secret_key", "oss.secret_key"},
62+
@ConnectorProperty(names = {"oss.hdfs.secret_key", "oss.secret_key", "dlf.secret_key", "dlf.catalog.secret_key"},
6363
sensitive = true,
6464
description = "The secret key of OSS.")
6565
protected String secretKey = "";
6666

67-
@ConnectorProperty(names = {"oss.hdfs.region", "dlf.region", "oss.region"},
67+
@ConnectorProperty(names = {"oss.hdfs.region", "oss.region", "dlf.region"},
6868
required = false,
6969
description = "The region of OSS.")
7070
protected String region;
@@ -86,8 +86,8 @@ public class OSSHdfsProperties extends HdfsCompatibleProperties {
8686
description = "The security token of OSS.")
8787
protected String securityToken = "";
8888

89-
private static final Set<String> OSS_ENDPOINT_KEY_NAME = ImmutableSet.of("oss.hdfs.endpoint",
90-
"dlf.endpoint", "dlf.catalog.endpoint", "oss.endpoint");
89+
private static final Set<String> OSS_ENDPOINT_KEY_NAME = ImmutableSet.of("oss.hdfs.endpoint", "oss.endpoint",
90+
"dlf.endpoint", "dlf.catalog.endpoint");
9191

9292
private Map<String, String> backendConfigProperties;
9393

@@ -111,21 +111,15 @@ public static boolean guessIsMe(Map<String, String> props) {
111111
}
112112
String endpoint = OSS_ENDPOINT_KEY_NAME.stream()
113113
.map(props::get)
114-
.filter(StringUtils::isNotBlank)
114+
.filter(ep -> StringUtils.isNotBlank(ep) && ep.endsWith(OSS_HDFS_ENDPOINT_SUFFIX))
115115
.findFirst()
116116
.orElse(null);
117-
if (StringUtils.isBlank(endpoint)) {
118-
return false;
119-
}
120-
return endpoint.endsWith(OSS_HDFS_ENDPOINT_SUFFIX) || endpoint.contains(DLF_ENDPOINT_KEY_WORDS);
117+
return StringUtils.isNotBlank(endpoint);
121118
}
122119

123120
@Override
124121
protected void checkRequiredProperties() {
125122
super.checkRequiredProperties();
126-
if (!isValidEndpoint(endpoint)) {
127-
throw new IllegalArgumentException("Property oss.endpoint is required and must be a valid OSS endpoint.");
128-
}
129123
}
130124

131125
private void convertDlfToOssEndpointIfNeeded() {
@@ -145,22 +139,9 @@ public static Optional<String> extractRegion(String endpoint) {
145139
return Optional.empty();
146140
}
147141

148-
public static boolean isValidEndpoint(String endpoint) {
149-
for (Pattern pattern : ENDPOINT_PATTERN) {
150-
if (pattern.matcher(endpoint.toLowerCase()).matches()) {
151-
return true;
152-
}
153-
}
154-
return false;
155-
}
156-
157142
@Override
158143
public void initNormalizeAndCheckProps() {
159144
super.initNormalizeAndCheckProps();
160-
if (!isValidEndpoint(endpoint.toLowerCase())) {
161-
throw new IllegalArgumentException("The endpoint is not a valid OSS HDFS endpoint: " + endpoint
162-
+ ". It should match the pattern: <region>.oss-dls.aliyuncs.com");
163-
}
164145
// Extract region from the endpoint, e.g., "cn-shanghai.oss-dls.aliyuncs.com" -> "cn-shanghai"
165146
if (StringUtils.isBlank(this.region)) {
166147
Optional<String> regionOptional = extractRegion(endpoint);
@@ -179,8 +160,6 @@ public void initNormalizeAndCheckProps() {
179160

180161
private static final String OSS_HDFS_ENDPOINT_SUFFIX = ".oss-dls.aliyuncs.com";
181162

182-
private static final String DLF_ENDPOINT_KEY_WORDS = "dlf";
183-
184163
@Override
185164
public Map<String, String> getBackendConfigProperties() {
186165
return backendConfigProperties;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,6 @@ public class OSSProperties extends AbstractS3CompatibleProperties {
148148
* - datalake.cn-hangzhou.aliyuncs.com => region = cn-hangzhou
149149
*/
150150
public static final Set<Pattern> ENDPOINT_PATTERN = ImmutableSet.of(STANDARD_ENDPOINT_PATTERN,
151-
Pattern.compile("(?:https?://)?([a-z]{2}-[a-z0-9-]+)\\.oss-dls\\.aliyuncs\\.com"),
152151
Pattern.compile("^(?:https?://)?dlf(?:-vpc)?\\.([a-z0-9-]+)\\.aliyuncs\\.com(?:/.*)?$"),
153152
Pattern.compile("^(?:https?://)?datalake(?:-vpc)?\\.([a-z0-9-]+)\\.aliyuncs\\.com(?:/.*)?$"));
154153

@@ -157,6 +156,8 @@ public class OSSProperties extends AbstractS3CompatibleProperties {
157156
private static List<String> DLF_TYPE_KEYWORDS = Arrays.asList("hive.metastore.type",
158157
"iceberg.catalog.type", "paimon.catalog.type");
159158

159+
private static final String DLS_URI_KEYWORDS = "oss-dls.aliyuncs";
160+
160161
protected OSSProperties(Map<String, String> origProps) {
161162
super(Type.OSS, origProps);
162163
}
@@ -177,6 +178,9 @@ protected static boolean guessIsMe(Map<String, String> origProps) {
177178
.findFirst()
178179
.orElse(null);
179180
if (StringUtils.isNotBlank(value)) {
181+
if (value.contains(DLS_URI_KEYWORDS)) {
182+
return false;
183+
}
180184
return (value.contains("aliyuncs.com"));
181185
}
182186

@@ -205,6 +209,10 @@ private static boolean isKnownObjectStorage(String value) {
205209
if (value == null) {
206210
return false;
207211
}
212+
boolean isDls = value.contains(DLS_URI_KEYWORDS);
213+
if (isDls) {
214+
return false;
215+
}
208216
if (value.startsWith("oss://")) {
209217
return true;
210218
}
@@ -213,8 +221,7 @@ private static boolean isKnownObjectStorage(String value) {
213221
}
214222
boolean isAliyunOss = (value.contains("oss-"));
215223
boolean isAmazonS3 = value.contains("s3.");
216-
boolean isDls = value.contains("dls");
217-
return isAliyunOss || isAmazonS3 || isDls;
224+
return isAliyunOss || isAmazonS3;
218225
}
219226

220227
private static boolean isDlfMSType(Map<String, String> params) {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -171,13 +171,22 @@ public static StorageProperties createPrimary(Map<String, String> origProps) {
171171
Arrays.asList(
172172
props -> (isFsSupport(props, FS_HDFS_SUPPORT)
173173
|| HdfsProperties.guessIsMe(props)) ? new HdfsProperties(props) : null,
174-
props -> ((isFsSupport(props, FS_OSS_HDFS_SUPPORT)
175-
|| isFsSupport(props, DEPRECATED_OSS_HDFS_SUPPORT))
176-
|| OSSHdfsProperties.guessIsMe(props)) ? new OSSHdfsProperties(props) : null,
174+
props -> {
175+
// OSS-HDFS and OSS are mutually exclusive - check OSS-HDFS first
176+
if ((isFsSupport(props, FS_OSS_HDFS_SUPPORT)
177+
|| isFsSupport(props, DEPRECATED_OSS_HDFS_SUPPORT))
178+
|| OSSHdfsProperties.guessIsMe(props)) {
179+
return new OSSHdfsProperties(props);
180+
}
181+
// Only check for regular OSS if OSS-HDFS is not enabled
182+
if (isFsSupport(props, FS_OSS_SUPPORT)
183+
|| OSSProperties.guessIsMe(props)) {
184+
return new OSSProperties(props);
185+
}
186+
return null;
187+
},
177188
props -> (isFsSupport(props, FS_S3_SUPPORT)
178189
|| S3Properties.guessIsMe(props)) ? new S3Properties(props) : null,
179-
props -> (isFsSupport(props, FS_OSS_SUPPORT)
180-
|| OSSProperties.guessIsMe(props)) ? new OSSProperties(props) : null,
181190
props -> (isFsSupport(props, FS_OBS_SUPPORT)
182191
|| OBSProperties.guessIsMe(props)) ? new OBSProperties(props) : null,
183192
props -> (isFsSupport(props, FS_COS_SUPPORT)

‎fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java‎

Lines changed: 58 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -112,18 +112,9 @@ public void testInvalidEndpoint() throws UserException {
112112
origProps.put("oss.endpoint", "cn-north-2-gov-1.oss-dls.aliyuncs.com");
113113
props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
114114
Assertions.assertEquals("cn-north-2-gov-1", props.getBackendConfigProperties().get("fs.oss.region"));
115-
origProps.put("dlf.endpoint", "dlf-vpc.cn-beijing.aliyuncs.com");
116-
props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
117-
Assertions.assertEquals("cn-beijing", props.getBackendConfigProperties().get("fs.oss.region"));
118115
origProps.remove("dlf.endpoint");
119116
props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
120117
Assertions.assertEquals("cn-north-2-gov-1", props.getBackendConfigProperties().get("fs.oss.region"));
121-
origProps.put("dlf.endpoint", "dlf-vpc.ap-southeast-5.aliyuncs.com");
122-
props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
123-
Assertions.assertEquals("ap-southeast-5", props.getBackendConfigProperties().get("fs.oss.region"));
124-
origProps.put("dlf.endpoint", "dlf.us-east-1.aliyuncs.com");
125-
props = (OSSHdfsProperties) StorageProperties.createAll(origProps).get(0);
126-
Assertions.assertEquals("us-east-1", props.getBackendConfigProperties().get("fs.oss.region"));
127118
}
128119

129120
@Test
@@ -167,4 +158,62 @@ public void testDefaultFS() {
167158
Assertions.assertEquals("oss://bucket", props.getBackendConfigProperties().get("fs.defaultFS"));
168159
}
169160

161+
@Test
162+
public void testOssAndOssHDFS() throws UserException {
163+
Map<String, String> origProps = new HashMap<>();
164+
origProps.put("oss.endpoint", "cn-shanghai.oss-dls.aliyuncs.com");
165+
origProps.put("oss.access_key", "testAccessKey");
166+
origProps.put("oss.secret_key", "testSecretKey");
167+
origProps.put("oss.hdfs.fs.defaultFS", "oss://my-bucket");
168+
origProps.put("dlf.endpoint", "cn-shanghai.oss-dlf.aliyuncs.com");
169+
origProps.put("dlf.access_key", "testAccessKey");
170+
origProps.put("dlf.secret_key", "testSecretKey");
171+
StorageProperties props = StorageProperties.createPrimary(origProps);
172+
Assertions.assertEquals("OSSHDFS", props.getStorageName());
173+
Assertions.assertEquals("oss://my-bucket", props.getBackendConfigProperties().get("fs.defaultFS"));
174+
Assertions.assertEquals("cn-shanghai", props.getBackendConfigProperties().get("fs.oss.region"));
175+
}
176+
177+
@Test
178+
public void testDlfAndOssHDFS() throws UserException {
179+
180+
Map<String, String> origProps = new HashMap<>();
181+
origProps.put("dlf.endpoint", "dlf-vpc.cn-beijing.aliyuncs.com");
182+
origProps.put("dlf.access_key", "testAccessKey");
183+
origProps.put("dlf.secret_key", "testSecretKey");
184+
origProps.put("oss.hdfs.enabled", "true");
185+
StorageProperties props = StorageProperties.createPrimary(origProps);
186+
Assertions.assertEquals("OSSHDFS", props.getStorageName());
187+
}
188+
189+
190+
@Test
191+
public void testOssHDFSNewProps() throws UserException {
192+
Map<String, String> origProps = new HashMap<>();
193+
origProps.put("oss.hdfs.endpoint", "cn-shanghai.oss-dls.aliyuncs.com");
194+
origProps.put("oss.hdfs.access_key", "testAccessKey");
195+
origProps.put("oss.hdfs.secret_key", "testSecretKey");
196+
StorageProperties props = StorageProperties.createPrimary(origProps);
197+
Assertions.assertEquals("OSSHDFS", props.getStorageName());
198+
Assertions.assertEquals("cn-shanghai", props.getBackendConfigProperties().get("fs.oss.region"));
199+
Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem",
200+
props.getBackendConfigProperties().get("fs.oss.impl"));
201+
origProps.put("oss.endpoint", "cn-shanghai.oss.aliyuncs.com");
202+
origProps.put("oss.access_key", "testAccessKey");
203+
origProps.put("oss.secret_key", "testSecretKey");
204+
props = StorageProperties.createPrimary(origProps);
205+
Assertions.assertEquals("OSSHDFS", props.getStorageName());
206+
Assertions.assertEquals("cn-shanghai", props.getBackendConfigProperties().get("fs.oss.region"));
207+
Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem",
208+
props.getBackendConfigProperties().get("fs.oss.impl"));
209+
origProps.put("dlf.endpoint", "dlf-vpc.cn-beijing.aliyuncs.com");
210+
origProps.put("dlf.access_key", "testAccessKey");
211+
origProps.put("dlf.secret_key", "testSecretKey");
212+
props = StorageProperties.createPrimary(origProps);
213+
Assertions.assertEquals("OSSHDFS", props.getStorageName());
214+
Assertions.assertEquals("cn-shanghai", props.getBackendConfigProperties().get("fs.oss.region"));
215+
Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem",
216+
props.getBackendConfigProperties().get("fs.oss.impl"));
217+
}
218+
170219
}

‎fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ public void testGetRegion() throws UserException {
120120
origProps.put("oss.endpoint", "http://s3.oss-cn-hongkong.aliyuncs.com");
121121
Assertions.assertEquals("cn-hongkong", ((OSSProperties) StorageProperties.createPrimary(origProps)).getRegion());
122122
origProps.put("oss.endpoint", "https://dlf.cn-beijing.aliyuncs.com");
123-
Assertions.assertEquals("cn-beijing", ((OSSProperties) StorageProperties.createAll(origProps).get(1)).getRegion());
123+
Assertions.assertEquals("cn-beijing", ((OSSProperties) StorageProperties.createAll(origProps).get(0)).getRegion());
124124
origProps.put("oss.endpoint", "datalake-vpc.cn-shenzhen.aliyuncs.com");
125125
Assertions.assertEquals("cn-shenzhen", ((OSSProperties) StorageProperties.createPrimary(origProps)).getRegion());
126126
origProps.put("oss.endpoint", "https://datalake-vpc.cn-shenzhen.aliyuncs.com");

0 commit comments

Comments
 (0)