Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.SdkSystemSetting;

import java.util.HashMap;
import java.util.Map;
Expand Down Expand Up @@ -250,6 +251,7 @@ public void initializeHadoopStorageConfig() {
// storage is OSS, OBS, etc., users may still configure the schema as "s3://".
// To ensure backward compatibility, we append S3-related properties by default.
appendS3HdfsProperties(hadoopStorageConfig);
setDefaultRequestChecksum();
}

private void appendS3HdfsProperties(Configuration hadoopStorageConfig) {
Expand All @@ -275,6 +277,34 @@ private void appendS3HdfsProperties(Configuration hadoopStorageConfig) {
hadoopStorageConfig.set("fs.s3a.path.style.access", getUsePathStyle());
}

/**
* Sets the AWS request checksum calculation property to "WHEN_REQUIRED"
* only if it has not been explicitly set by the user.
*
* <p>
* Background:
* AWS SDK for Java v2 uses the system property
* {@link SdkSystemSetting#AWS_REQUEST_CHECKSUM_CALCULATION} to determine
* whether request payloads should have a checksum calculated.
* <p>
* According to the official AWS discussion:
* https://github.com/aws/aws-sdk-java-v2/discussions/5802
* - Default SDK behavior may calculate checksums automatically if the property is not set.
* - Automatic calculation can affect performance or cause unexpected behavior for large requests.
* <p>
* This method ensures:
* 1. The property is set to "WHEN_REQUIRED" only if the user has not already set it.
* 2. User-specified settings are never overridden.
* 3. Aligns with AWS SDK recommended best practices.
* </p>
*/
public static void setDefaultRequestChecksum() {
String key = SdkSystemSetting.AWS_REQUEST_CHECKSUM_CALCULATION.property();
if (System.getProperty(key) == null) {
System.setProperty(key, "WHEN_REQUIRED");
}
}

@Override
public String getStorageName() {
return "S3";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.SdkSystemSetting;

import java.util.HashMap;
import java.util.Map;
Expand Down Expand Up @@ -268,4 +269,13 @@ public void testS3DisableHadoopCache() throws UserException {
Assertions.assertFalse(s3Properties.hadoopStorageConfig.getBoolean("fs.oss.impl.disable.cache", false));
}

@Test
public void testResuestCheckSum() throws UserException {
Map<String, String> props = Maps.newHashMap();
props.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com");
Assertions.assertEquals("WHEN_REQUIRED", System.getProperty(SdkSystemSetting.AWS_REQUEST_CHECKSUM_CALCULATION.property()));
System.setProperty("aws.requestChecksumCalculation", "ALWAYS");
Assertions.assertEquals("ALWAYS", System.getProperty(SdkSystemSetting.AWS_REQUEST_CHECKSUM_CALCULATION.property()));
}

}
Loading