Skip to content

Commit 76bce38

Browse files
authored
meta/redis: support client cache (#6495)
Signed-off-by: jiefenghuang <[email protected]>
1 parent adf8c16 commit 76bce38

File tree

7 files changed

+606
-4
lines changed

7 files changed

+606
-4
lines changed

docs/en/reference/redis-csc.md

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# Redis Client-Side Caching Support in JuiceFS
2+
3+
Starting with version 6.0, Redis provides [Client-Side Caching](https://redis.io/docs/latest/develop/reference/client-side-caching) which allows clients to maintain local caches of data in a faster and more efficient way. JuiceFS includes full support for this feature, offering significant performance improvements for metadata operations.
4+
5+
## How it works
6+
7+
Redis Client-Side Caching (CSC) works by:
8+
9+
1. The client enables tracking mode with `CLIENT TRACKING ON BCAST`
10+
2. The client caches data locally after reading it from Redis
11+
3. Redis notifies the client when cached keys are modified by any client
12+
4. The client invalidates those keys in its local cache
13+
14+
This results in reduced network traffic, lower latency, and higher throughput.
15+
16+
## Configuration
17+
18+
JuiceFS supports Redis CSC through the following options in the metadata URL:
19+
20+
```shell
21+
--meta-url="redis://localhost/1?client-cache=true" # Enable client-side caching (always BCAST mode)
22+
--meta-url="redis://localhost/1?client-cache=true&client-cache-size=500" # Set cache size (default 12800)
23+
--meta-url="redis://localhost/1?client-cache=true&client-cache-expire=60s" # Set cache expiration (default: 60s)
24+
```
25+
26+
### Options
27+
28+
- `client-cache`: Enables client-side caching in BCAST mode (set to any value except "false")
29+
- `client-cache-size`: Maximum cache size (default: 12800)
30+
- `client-cache-expire`: Cache expiration time (default: 60s)
31+
- `client-cache-preload`: Number of file objects under the root directory preloaded after mounting. (default: 0)
32+
33+
When client-side caching is enabled, JuiceFS caches:
34+
35+
1. **Inode attributes**: File/directory metadata like permissions, size, timestamps
36+
2. **Directory entries**: Name to inode mappings for faster lookups
37+
38+
> **Note:** Redis Client Side Cache requires Redis server version 6.0 or higher. Using this feature with older Redis versions will result in errors.
39+
40+
### Preloading Cache
41+
42+
When client-side caching is enabled and `client-cache-preload` is set, JuiceFS will preload the file-object attributes and entries under the root directory after mounting. This lazy preloading happens in the background and helps to:
43+
44+
1. Warm up the cache for common operations
45+
2. Reduce latency for initial file system operations
46+
3. Provide better performance from the moment the file system is mounted
47+
48+
The preloading process intelligently prioritizes the most important inodes by:
49+
50+
1. Starting with the root directory
51+
2. Loading the most frequently accessed top-level directories and files
52+
3. Recursively exploring important subdirectories
53+
54+
The preloading process runs in a background goroutine with fail-safe mechanisms and won't block or affect normal file system operations.
55+
56+
## Modes
57+
58+
JuiceFS uses BCAST mode for simplicity and reliability:
59+
60+
- **BCAST mode**: All keys accessed by the client are tracked and notifications are sent for any changes.
61+
62+
BCAST mode provides the simplest implementation while ensuring cache coherence across all clients.
63+
64+
## Requirements
65+
66+
- Redis server version 6.0 or higher
67+
- JuiceFS with CSC support enabled
68+
69+
## Performance Considerations
70+
71+
1. The default 12800 cache size should be sufficient for most workloads
72+
2. For very large filesystems with millions of files, you may benefit from increasing the cache size
73+
3. The cache is most effective for metadata-heavy workloads with many repeated operations
74+
4. For very write-heavy workloads, consider disabling CSC as invalidation traffic may offset benefits
75+
76+
## Troubleshooting
77+
78+
If you experience crashes or instability with CSC enabled:
79+
80+
1. Update to the latest JuiceFS version which contains important fixes for CSC
81+
2. Try reducing the cache size with `client-cache-size`
82+
3. Check Redis server logs for any memory or client tracking issues
83+
4. Make sure your Redis server version is 6.0 or higher
84+
5. If problems persist, disable CSC by removing the `client-cache` parameter
85+
86+
JuiceFS includes robust error handling for various Redis CSC-specific responses to ensure stable operation even when Redis sends unexpected response formats due to client tracking.
87+
88+
## References
89+
90+
- [Redis Client-Side Caching Documentation](https://redis.io/docs/latest/develop/reference/client-side-caching)

go.mod

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ require (
4141
github.com/hanwen/go-fuse/v2 v2.1.1-0.20210611132105-24a1dfe6b4f8
4242
github.com/hashicorp/consul/api v1.29.2
4343
github.com/hashicorp/go-hclog v1.6.3
44+
github.com/hashicorp/golang-lru/v2 v2.0.7
4445
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible
4546
github.com/hungys/go-lz4 v0.0.0-20170805124057-19ff7f07f099
4647
github.com/jackc/pgx/v5 v5.7.3
@@ -68,7 +69,7 @@ require (
6869
github.com/prometheus/prometheus v0.54.1
6970
github.com/qingstor/qingstor-sdk-go/v4 v4.4.0
7071
github.com/qiniu/go-sdk/v7 v7.25.2
71-
github.com/redis/go-redis/v9 v9.7.3
72+
github.com/redis/go-redis/v9 v9.16.0
7273
github.com/sirupsen/logrus v1.9.3
7374
github.com/smartystreets/goconvey v1.7.2
7475
github.com/spf13/cast v1.7.1
@@ -354,3 +355,5 @@ replace github.com/mattn/go-colorable v0.1.9 => github.com/juicedata/go-colorabl
354355
replace github.com/mattn/go-colorable v0.0.9 => github.com/juicedata/go-colorable v0.0.0-20250208072043-a97a0c2023db
355356

356357
replace github.com/cloudsoda/go-smb2 => github.com/juicedata/go-smb2 v0.0.0-20250917090526-d2d0abfb0e05
358+
359+
replace github.com/hashicorp/golang-lru/v2 v2.0.7 => github.com/juicedata/golang-lru/v2 v2.0.8-0.20251126062551-1b321869f904

go.sum

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -487,6 +487,8 @@ github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d h1:kpQMvNZJKGY3
487487
github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d/go.mod h1:dlxKkLh3qAIPtgr2U/RVzsZJDuXA1ffg+Njikfmhvgw=
488488
github.com/juicedata/gogfapi v0.0.0-20241204082332-ecd102647f80 h1:EPg/f3lhbAOjE2M0WpVi47Fk62mEmmPejRuGVdOFQww=
489489
github.com/juicedata/gogfapi v0.0.0-20241204082332-ecd102647f80/go.mod h1:Ho5G4KgrgbMKW0buAJdOmYoJcOImkzznJQaLiATrsx4=
490+
github.com/juicedata/golang-lru/v2 v2.0.8-0.20251126062551-1b321869f904 h1:oNtkL1jwrNMMcBlHNW1fhdl4quK7p1EdR7o1Rja5xpM=
491+
github.com/juicedata/golang-lru/v2 v2.0.8-0.20251126062551-1b321869f904/go.mod h1:qnbgnNzfydwuHjSCApF4bdul+tZ8T3y1MkZG/OFczLA=
490492
github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible h1:2/ttSmYoX+QMegpNyAJR0Y6aHcVk57F7RJit5xN2T/s=
491493
github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible/go.mod h1:Ukwa8ffRQLV6QRwpqGioPjn2Wnf7TBDA4DbennDOqHE=
492494
github.com/juicedata/minio v0.0.0-20251120043259-079fa6a601db h1:yGKlGEz3nOD2IovjI+V4O+eY1TPgOp/T6gOxMl9/xKI=
@@ -701,8 +703,8 @@ github.com/qiniu/go-sdk/v7 v7.25.2/go.mod h1:dmKtJ2ahhPWFVi9o1D5GemmWoh/ctuB9peq
701703
github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs=
702704
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
703705
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
704-
github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM=
705-
github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA=
706+
github.com/redis/go-redis/v9 v9.16.0 h1:OotgqgLSRCmzfqChbQyG1PHC3tLNR89DG4jdOERSEP4=
707+
github.com/redis/go-redis/v9 v9.16.0/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
706708
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
707709
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
708710
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=

pkg/meta/redis.go

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ import (
5050
"github.com/juicedata/juicefs/pkg/utils"
5151
"github.com/pkg/errors"
5252
"github.com/redis/go-redis/v9"
53+
"github.com/redis/go-redis/v9/maintnotifications"
5354
"golang.org/x/sync/errgroup"
5455
)
5556

@@ -92,6 +93,7 @@ type redisMeta struct {
9293
prefix string
9394
shaLookup string // The SHA returned by Redis for the loaded `scriptLookup`
9495
shaResolve string // The SHA returned by Redis for the loaded `scriptResolve`
96+
cache *redisCache
9597
}
9698

9799
var _ Meta = (*redisMeta)(nil)
@@ -122,6 +124,14 @@ func newRedisMeta(driver, addr string, conf *Config) (Meta, error) {
122124
keyFile := query.pop("tls-key-file")
123125
caCertFile := query.pop("tls-ca-cert-file")
124126
tlsServerName := query.pop("tls-server-name")
127+
128+
// Client-side caching options
129+
clientCacheStr := query.pop("client-cache")
130+
clientCache := clientCacheStr != "false" && clientCacheStr != ""
131+
clientCacheSize := query.getInt("client-cache-size", "client_cache_size", 12800)
132+
// Default TTL to prevent reading stale cache for a long time when the connection fails.
133+
clientCacheExpiry := query.duration("client-cache-expire", "client_cache_expire", time.Minute)
134+
clientCachePreload := query.getInt("client-cache-preload", "client_cache_preload", 0) // may cause conflict
125135
u.RawQuery = values.Encode()
126136

127137
hosts := u.Host
@@ -173,8 +183,10 @@ func newRedisMeta(driver, addr string, conf *Config) (Meta, error) {
173183
opt.MaxRetryBackoff = maxRetryBackoff
174184
opt.ReadTimeout = readTimeout
175185
opt.WriteTimeout = writeTimeout
176-
var rdb redis.UniversalClient
186+
opt.MaintNotificationsConfig = &maintnotifications.Config{Mode: maintnotifications.ModeDisabled}
177187
var prefix string
188+
var rdb redis.UniversalClient
189+
178190
if strings.Contains(hosts, ",") && strings.Index(hosts, ",") < strings.Index(hosts, ":") {
179191
var fopt redis.FailoverOptions
180192
ps := strings.Split(hosts, ",")
@@ -269,15 +281,37 @@ func newRedisMeta(driver, addr string, conf *Config) (Meta, error) {
269281
rdb: rdb,
270282
prefix: prefix,
271283
}
284+
if clientCache {
285+
m.cache = newRedisCache(prefix, clientCacheSize, clientCacheExpiry, clientCachePreload)
286+
if err = m.cache.init(m.rdb); err != nil {
287+
logger.Warnf("Failed to setup client-side caching: %v", err)
288+
m.cache = nil
289+
}
290+
}
272291
m.en = m
273292
m.checkServerConfig()
274293
return m, nil
275294
}
276295

277296
func (m *redisMeta) Shutdown() error {
297+
if m.cache != nil {
298+
m.cache.close()
299+
m.cache = nil
300+
}
278301
return m.rdb.Close()
279302
}
280303

304+
// Override NewSession to initialize client-side cache after session is created
305+
func (m *redisMeta) NewSession(record bool) error {
306+
// First, create the session normally
307+
err := m.baseMeta.NewSession(record)
308+
if err != nil {
309+
return err
310+
}
311+
go m.preloadCache()
312+
return nil
313+
}
314+
281315
func (m *redisMeta) doDeleteSlice(id uint64, size uint32) error {
282316
return m.rdb.HDel(Background(), m.sliceRefs(), m.sliceKey(id, size)).Err()
283317
}
@@ -919,6 +953,20 @@ func (m *redisMeta) doLookup(ctx Context, parent Ino, name string, inode *Ino, a
919953
var encodedAttr []byte
920954
var err error
921955
entryKey := m.entryKey(parent)
956+
if m.cache != nil {
957+
if entry, ok := m.cache.entryCache.Get(m.cache.entryName(parent, name)); ok {
958+
if !entry.isMark() {
959+
*inode = entry.ino
960+
if attr != nil {
961+
*attr = entry.Attr
962+
}
963+
return 0
964+
}
965+
m.cache.entryCache.AddIf(m.cache.entryName(parent, name), &entryMark, func(oldEntry *cachedEntry, exists bool) bool {
966+
return exists
967+
})
968+
}
969+
}
922970
if len(m.shaLookup) > 0 && attr != nil && !m.conf.CaseInsensi && m.prefix == "" {
923971
var res interface{}
924972
var returnedIno int64
@@ -946,6 +994,13 @@ func (m *redisMeta) doLookup(ctx Context, parent Ino, name string, inode *Ino, a
946994
if err == nil {
947995
m.parseAttr(encodedAttr, attr)
948996
m.of.Update(foundIno, attr)
997+
if m.cache != nil {
998+
ce := &cachedEntry{ino: foundIno}
999+
m.parseAttr(encodedAttr, &ce.Attr)
1000+
_, _ = m.cache.entryCache.AddIf(m.cache.entryName(parent, name), ce, func(oldEntry *cachedEntry, exists bool) bool {
1001+
return exists && oldEntry.isMark()
1002+
})
1003+
}
9491004
} else if err == redis.Nil { // corrupt entry
9501005
logger.Warnf("no attribute for inode %d (%d, %s)", foundIno, parent, name)
9511006
*attr = Attr{Typ: foundType}

0 commit comments

Comments
 (0)