Skip to content

Commit ec44d1a

Browse files
authored
gateway: reduce memory for list objects (#6476)
1 parent 5573197 commit ec44d1a

File tree

3 files changed

+42
-25
lines changed

3 files changed

+42
-25
lines changed

go.mod

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,6 @@ require (
163163
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
164164
github.com/dchest/siphash v1.2.1 // indirect
165165
github.com/dgraph-io/ristretto/v2 v2.1.0 // indirect
166-
github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
167166
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
168167
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
169168
github.com/djherbis/atime v1.0.0 // indirect
@@ -189,7 +188,7 @@ require (
189188
github.com/go-playground/validator/v10 v10.19.0 // indirect
190189
github.com/go-resty/resty/v2 v2.13.1 // indirect
191190
github.com/gogo/protobuf v1.3.2 // indirect
192-
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
191+
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
193192
github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
194193
github.com/golang/glog v1.2.2 // indirect
195194
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -326,7 +325,7 @@ require (
326325
xorm.io/builder v0.3.7 // indirect
327326
)
328327

329-
replace github.com/minio/minio v0.0.0-20210206053228-97fe57bba92c => github.com/juicedata/minio v0.0.0-20250321080125-0f92d5f311b8
328+
replace github.com/minio/minio v0.0.0-20210206053228-97fe57bba92c => github.com/juicedata/minio v0.0.0-20250825085608-abcb3b7bc326
330329

331330
replace github.com/hanwen/go-fuse/v2 v2.1.1-0.20210611132105-24a1dfe6b4f8 => github.com/juicedata/go-fuse/v2 v2.1.1-0.20250807045235-112198daa7df
332331

go.sum

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -306,10 +306,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
306306
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
307307
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
308308
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
309-
github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
310-
github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
311-
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
312-
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
309+
github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
310+
github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
313311
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
314312
github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
315313
github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
@@ -491,8 +489,8 @@ github.com/juicedata/gogfapi v0.0.0-20241204082332-ecd102647f80 h1:EPg/f3lhbAOjE
491489
github.com/juicedata/gogfapi v0.0.0-20241204082332-ecd102647f80/go.mod h1:Ho5G4KgrgbMKW0buAJdOmYoJcOImkzznJQaLiATrsx4=
492490
github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible h1:2/ttSmYoX+QMegpNyAJR0Y6aHcVk57F7RJit5xN2T/s=
493491
github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible/go.mod h1:Ukwa8ffRQLV6QRwpqGioPjn2Wnf7TBDA4DbennDOqHE=
494-
github.com/juicedata/minio v0.0.0-20250321080125-0f92d5f311b8 h1:koC8S1UrKWlMUGn2en2Ol1gdIsr78hyg0CbZOEpdT8w=
495-
github.com/juicedata/minio v0.0.0-20250321080125-0f92d5f311b8/go.mod h1:UOWyfa3ls1tnpJrNw2yzGqfrwM4nzsZq/qz+zd6H+/Q=
492+
github.com/juicedata/minio v0.0.0-20250825085608-abcb3b7bc326 h1:+iT+70JZhX+6CwlYe+0qoTuRGts5KSL/UMLh8tOMa4A=
493+
github.com/juicedata/minio v0.0.0-20250825085608-abcb3b7bc326/go.mod h1:1/4WHQKDOsWA1dd3ADrq9IE/jtFec9MHLy656kIXjNg=
496494
github.com/juicedata/mpb/v7 v7.0.4-0.20231024073412-2b8d31be510b h1:0/6suPNZnrOlRlBaU/Bnitu8HiKkkLSzQhHbwQ9AysM=
497495
github.com/juicedata/mpb/v7 v7.0.4-0.20231024073412-2b8d31be510b/go.mod h1:NXGsfPGx6G2JssqvEcULtDqUrxuuYs4llpv8W6ZUpzk=
498496
github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI=

pkg/gateway/gateway.go

Lines changed: 36 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ type Config struct {
7070

7171
func NewJFSGateway(jfs *fs.FileSystem, conf *vfs.Config, gConf *Config) (minio.ObjectLayer, error) {
7272
mctx = meta.NewContext(uint32(os.Getpid()), uint32(os.Getuid()), []uint32{uint32(os.Getgid())})
73-
jfsObj := &jfsObjects{fs: jfs, conf: conf, listPool: minio.NewTreeWalkPool(time.Minute * 30), gConf: gConf, nsMutex: minio.NewNSLock(false)}
73+
jfsObj := &jfsObjects{fs: jfs, conf: conf, listPool: minio.NewTreeWalkPool(time.Second * 10), gConf: gConf, nsMutex: minio.NewNSLock(false)}
7474
go jfsObj.cleanup()
7575
return jfsObj, nil
7676
}
@@ -339,21 +339,10 @@ func (n *jfsObjects) listDirFactory() minio.ListDirFunc {
339339
continue
340340
}
341341
}
342-
entry := &minio.Entry{Name: fi.Name(),
343-
Info: &minio.ObjectInfo{
344-
Bucket: bucket,
345-
Name: fi.Name(),
346-
ModTime: fi.ModTime(),
347-
Size: fi.Size(),
348-
IsDir: fi.IsDir(),
349-
AccTime: fi.ModTime(),
350-
IsLatest: true,
351-
},
352-
}
342+
entry := &minio.Entry{Name: fi.Name(), Info: fi}
353343

354344
if fi.IsDir() {
355345
entry.Name += sep
356-
entry.Info.Size = 0
357346
}
358347
entries = append(entries, entry)
359348
}
@@ -383,9 +372,10 @@ func (n *jfsObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
383372
if err := n.checkBucket(ctx, bucket); err != nil {
384373
return loi, err
385374
}
386-
getObjectInfo := func(ctx context.Context, bucket, object string, info *minio.ObjectInfo) (obj minio.ObjectInfo, err error) {
375+
getObjectInfo := func(ctx context.Context, bucket, object string, fi_ any) (obj minio.ObjectInfo, err error) {
387376
var eno syscall.Errno
388-
if info == nil {
377+
var info *minio.ObjectInfo
378+
if fi_ == nil {
389379
var fi *fs.FileStat
390380
fi, eno = n.fs.Stat(mctx, n.path(bucket, object))
391381
if eno == 0 {
@@ -416,6 +406,20 @@ func (n *jfsObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
416406
}
417407
eno = 0
418408
}
409+
} else {
410+
fi := fi_.(*fs.FileStat)
411+
info = &minio.ObjectInfo{
412+
Bucket: bucket,
413+
Name: fi.Name(),
414+
ModTime: fi.ModTime(),
415+
Size: fi.Size(),
416+
IsDir: fi.IsDir(),
417+
AccTime: fi.ModTime(),
418+
IsLatest: true,
419+
}
420+
if fi.IsDir() {
421+
info.Size = 0
422+
}
419423
}
420424

421425
if info == nil {
@@ -1457,7 +1461,23 @@ func (n *jfsObjects) ListObjectVersions(ctx context.Context, bucket, prefix, mar
14571461
return loi, err
14581462
}
14591463

1460-
func (n *jfsObjects) getObjectInfoNoFSLock(ctx context.Context, bucket, object string, info *minio.ObjectInfo) (oi minio.ObjectInfo, e error) {
1464+
func (n *jfsObjects) getObjectInfoNoFSLock(ctx context.Context, bucket, object string, info any) (oi minio.ObjectInfo, e error) {
1465+
if info != nil {
1466+
fi := info.(*fs.FileStat)
1467+
oi = minio.ObjectInfo{
1468+
Bucket: bucket,
1469+
Name: fi.Name(),
1470+
ModTime: fi.ModTime(),
1471+
Size: fi.Size(),
1472+
IsDir: fi.IsDir(),
1473+
AccTime: fi.ModTime(),
1474+
IsLatest: true,
1475+
}
1476+
if fi.IsDir() {
1477+
oi.Size = 0
1478+
}
1479+
return
1480+
}
14611481
return n.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
14621482
}
14631483

0 commit comments

Comments
 (0)