Skip to content

fix: sync command region handling and bucket region auto-detection #858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gchait wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix: sync command region handling and bucket region auto-detection #858

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions command/sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -301,12 +301,22 @@ func compareObjects(sourceObjects, destObjects chan *storage.Object, isSrcBatch
// given URLs. The returned channels gives objects sorted in ascending order
// with respect to their url.Relative path. See also storage.Less.
func (s Sync) getSourceAndDestinationObjects(ctx context.Context, cancel context.CancelFunc, srcurl, dsturl *url.URL) (chan *storage.Object, chan *storage.Object, error) {
sourceClient, err := storage.NewClient(ctx, srcurl, s.storageOpts)
// Create source client with source region
srcOpts := s.storageOpts
if s.srcRegion != "" {
srcOpts.SetRegion(s.srcRegion)
}
sourceClient, err := storage.NewClient(ctx, srcurl, srcOpts)
if err != nil {
return nil, nil, err
}

destClient, err := storage.NewClient(ctx, dsturl, s.storageOpts)
// Create destination client with destination region
dstOpts := s.storageOpts
if s.dstRegion != "" {
dstOpts.SetRegion(s.dstRegion)
}
destClient, err := storage.NewClient(ctx, dsturl, dstOpts)
if err != nil {
return nil, nil, err
}
Expand Down
24 changes: 14 additions & 10 deletions storage/s3.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1334,20 +1334,18 @@ func (sc *SessionCache) clear() {
}

func setSessionRegion(ctx context.Context, sess *session.Session, bucket string) error {
region := aws.StringValue(sess.Config.Region)

if region != "" {
return nil
}

// set default region
sess.Config.Region = aws.String(endpoints.UsEast1RegionID)

// If no bucket specified, use default or configured region
if bucket == "" {
region := aws.StringValue(sess.Config.Region)
if region == "" {
// set default region for bucket-less operations
sess.Config.Region = aws.String(endpoints.UsEast1RegionID)
}
return nil
}

// auto-detection
// Always auto-detect bucket region when bucket is specified
// This makes behavior consistent with AWS CLI
region, err := s3manager.GetBucketRegion(ctx, sess, bucket, "", func(r *request.Request) {
// s3manager.GetBucketRegion uses Path style addressing and
// AnonymousCredentials by default, updating Request's Config to match
Expand All @@ -1364,6 +1362,12 @@ func setSessionRegion(ctx context.Context, sess *session.Session, bucket string)
err = fmt.Errorf("session: fetching region failed: %v", err)
msg := log.ErrorMessage{Err: err.Error()}
log.Error(msg)

// Fall back to configured region if auto-detection fails
configuredRegion := aws.StringValue(sess.Config.Region)
if configuredRegion == "" {
sess.Config.Region = aws.String(endpoints.UsEast1RegionID)
}
} else {
sess.Config.Region = aws.String(region)
}
Expand Down
Loading