Skip to content

Commit 67ca95c

Browse files
jiefenghuangjiefenghuang
committed
review
Signed-off-by: jiefenghuang <[email protected]>
1 parent 4e84c1a commit 67ca95c

File tree

4 files changed

+57
-37
lines changed

4 files changed

+57
-37
lines changed

‎cmd/gc.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,11 @@ $ juicefs gc redis://localhost --delete`,
7070
Value: 10,
7171
Usage: "number threads to delete leaked objects",
7272
},
73+
&cli.BoolFlag{
74+
Name: "scan-dangling",
75+
Value: false,
76+
Usage: "Scan dangling objects (nodes/edges...), better in read-only mode (time-intensive).",
77+
},
7378
},
7479
}
7580
}
@@ -196,7 +201,7 @@ func gc(ctx *cli.Context) error {
196201

197202
// List all slices in metadata engine
198203
slices := make(map[meta.Ino][]meta.Slice)
199-
r := m.ListSlices(c, slices, true, true, delete, sliceCSpin.Increment)
204+
r := m.ListSlices(c, slices, true, ctx.Bool("scan-dangling"), delete, sliceCSpin.Increment)
200205
if r != 0 {
201206
logger.Fatalf("list all slices: %s", r)
202207
}

‎pkg/meta/redis.go‎

Lines changed: 24 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3058,9 +3058,9 @@ func (m *redisMeta) cleanupLeakedInodesAndEdges(clean bool, before time.Duration
30583058
Name string
30593059
}
30603060
var ctx = Background()
3061-
var foundInodes = make(map[Ino]*entry)
3062-
foundInodes[RootInode] = &entry{Parent: RootInode, Name: "/"}
3063-
foundInodes[TrashInode] = &entry{Parent: RootInode, Name: ".trash"}
3061+
var node2Edges = make(map[Ino][]*entry)
3062+
node2Edges[RootInode] = []*entry{{Parent: RootInode, Name: "/"}}
3063+
node2Edges[TrashInode] = []*entry{{Parent: RootInode, Name: ".trash"}}
30643064
cutoff := time.Now().Add(-before)
30653065
prefix := len(m.prefix)
30663066

@@ -3074,7 +3074,7 @@ func (m *redisMeta) cleanupLeakedInodesAndEdges(clean bool, before time.Duration
30743074
return eno
30753075
}
30763076
for _, e := range entries {
3077-
foundInodes[e.Inode] = &entry{Parent: Ino(ino), Name: string(e.Name)}
3077+
node2Edges[e.Inode] = append(node2Edges[e.Inode], &entry{Parent: Ino(ino), Name: string(e.Name)})
30783078
}
30793079
}
30803080
return nil
@@ -3096,7 +3096,7 @@ func (m *redisMeta) cleanupLeakedInodesAndEdges(clean bool, before time.Duration
30963096
var attr Attr
30973097
m.parseAttr([]byte(v.(string)), &attr)
30983098
ino, _ := strconv.Atoi(keys[i][prefix+1:])
3099-
if _, ok := foundInodes[Ino(ino)]; !ok && time.Unix(attr.Ctime, 0).Before(cutoff) {
3099+
if _, ok := node2Edges[Ino(ino)]; !ok && time.Unix(attr.Ctime, 0).Before(cutoff) {
31003100
logger.Infof("found leaded inode: %d %+v", ino, attr)
31013101
if clean {
31023102
err = m.doDeleteSustainedInode(0, Ino(ino))
@@ -3105,27 +3105,33 @@ func (m *redisMeta) cleanupLeakedInodesAndEdges(clean bool, before time.Duration
31053105
}
31063106
}
31073107
}
3108-
foundInodes[Ino(ino)] = nil
3108+
node2Edges[Ino(ino)] = nil
31093109
}
31103110
return nil
31113111
}); err != nil {
31123112
logger.Errorf("scan inodes: %s", err)
31133113
return
31143114
}
31153115

3116-
foundInodes[RootInode], foundInodes[TrashInode] = nil, nil
3117-
pipe := m.rdb.Pipeline()
3118-
for c, e := range foundInodes {
3119-
if e != nil {
3120-
logger.Infof("found leaked edge %d -> (%d, %s)", e.Parent, c, e.Name)
3121-
if clean {
3122-
pipe.HDel(ctx, m.entryKey(e.Parent), e.Name)
3116+
node2Edges[RootInode], node2Edges[TrashInode] = nil, nil
3117+
for c, es := range node2Edges {
3118+
if err := m.txn(ctx, func(tx *redis.Tx) error {
3119+
// double check
3120+
if err := tx.Get(ctx, m.inodeKey(c)).Err(); err == nil {
3121+
return nil
3122+
} else if err != redis.Nil {
3123+
return err
31233124
}
3124-
}
3125-
}
3126-
if pipe.Len() > 0 {
3127-
if _, err := pipe.Exec(ctx); err != nil {
3128-
logger.Errorf("delete leaked edges: %s", err)
3125+
for _, e := range es {
3126+
logger.Infof("found leaked edge %d -> (%d, %s)", e.Parent, c, e.Name)
3127+
if clean {
3128+
tx.HDel(ctx, m.entryKey(e.Parent), e.Name)
3129+
}
3130+
}
3131+
return nil
3132+
}, m.inodeKey(c)); err != nil {
3133+
logger.Errorf("delete leaked edges %d: %s", c, err)
3134+
return
31293135
}
31303136
}
31313137
}

‎pkg/meta/sql.go‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3154,9 +3154,9 @@ func (m *dbMeta) cleanupLeakedInodesAndEdges(clean bool, before time.Duration) {
31543154
Parent Ino
31553155
Name []byte
31563156
}
3157-
var foundInodes = make(map[Ino]*entry)
3158-
foundInodes[RootInode] = &entry{Parent: RootInode, Name: []byte("/")}
3159-
foundInodes[TrashInode] = &entry{Parent: RootInode, Name: []byte(".trash")}
3157+
var node2Edges = make(map[Ino][]*entry)
3158+
node2Edges[RootInode] = []*entry{{Parent: RootInode, Name: []byte("/")}}
3159+
node2Edges[TrashInode] = []*entry{{Parent: RootInode, Name: []byte(".trash")}}
31603160
cutoff := time.Now().Add(-before)
31613161

31623162
var edges []edge
@@ -3166,7 +3166,7 @@ func (m *dbMeta) cleanupLeakedInodesAndEdges(clean bool, before time.Duration) {
31663166
return err
31673167
}
31683168
for _, edge := range edges {
3169-
foundInodes[edge.Inode] = &entry{Parent: edge.Parent, Name: edge.Name}
3169+
node2Edges[edge.Inode] = append(node2Edges[edge.Inode], &entry{Parent: edge.Parent, Name: edge.Name})
31703170
}
31713171
if err := s.Find(&nodes); err != nil {
31723172
return err
@@ -3178,7 +3178,7 @@ func (m *dbMeta) cleanupLeakedInodesAndEdges(clean bool, before time.Duration) {
31783178
}
31793179

31803180
for _, node := range nodes {
3181-
if _, ok := foundInodes[node.Inode]; !ok && time.Unix(node.Ctime/1e6, 0).Before(cutoff) {
3181+
if _, ok := node2Edges[node.Inode]; !ok && time.Unix(node.Ctime/1e6, 0).Before(cutoff) {
31823182
logger.Infof("found leaded inode: %d %+v", node.Inode, node)
31833183
if clean {
31843184
err := m.doDeleteSustainedInode(0, node.Inode)
@@ -3187,13 +3187,19 @@ func (m *dbMeta) cleanupLeakedInodesAndEdges(clean bool, before time.Duration) {
31873187
}
31883188
}
31893189
}
3190-
foundInodes[node.Inode] = nil
3190+
node2Edges[node.Inode] = nil
31913191
}
31923192

3193-
foundInodes[RootInode], foundInodes[TrashInode] = nil, nil
3193+
node2Edges[RootInode], node2Edges[TrashInode] = nil, nil
31943194
if err := m.txn(func(s *xorm.Session) error {
3195-
for c, e := range foundInodes {
3196-
if e != nil {
3195+
for c, es := range node2Edges {
3196+
n := node{Inode: c}
3197+
if exist, err := s.Get(&n); err != nil {
3198+
return err
3199+
} else if exist {
3200+
continue
3201+
}
3202+
for _, e := range es {
31973203
logger.Infof("found leaked edge %d -> (%d, %s)", e.Parent, c, e.Name)
31983204
if clean {
31993205
if _, err := s.Delete(&edge{Parent: e.Parent, Name: e.Name}); err != nil {

‎pkg/meta/tkv.go‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2444,9 +2444,9 @@ func (m *kvMeta) cleanupLeakedInodesAndEdges(clean bool, before time.Duration) {
24442444
Parent Ino
24452445
Name []byte
24462446
}
2447-
var foundInodes = make(map[Ino]*entry)
2448-
foundInodes[RootInode] = &entry{Parent: RootInode, Name: []byte("/")}
2449-
foundInodes[TrashInode] = &entry{Parent: RootInode, Name: []byte(".trash")}
2447+
var node2Edges = make(map[Ino][]*entry)
2448+
node2Edges[RootInode] = []*entry{{Parent: RootInode, Name: []byte("/")}}
2449+
node2Edges[TrashInode] = []*entry{{Parent: RootInode, Name: []byte(".trash")}}
24502450
cutoff := time.Now().Add(-before)
24512451

24522452
nodes := make(map[Ino]*Attr)
@@ -2461,7 +2461,7 @@ func (m *kvMeta) cleanupLeakedInodesAndEdges(clean bool, before time.Duration) {
24612461
case 'D':
24622462
name := string(key[10:])
24632463
_, inode := m.parseEntry(value)
2464-
foundInodes[inode] = &entry{Parent: ino, Name: []byte(name)}
2464+
node2Edges[inode] = append(node2Edges[inode], &entry{Parent: ino, Name: []byte(name)})
24652465
}
24662466
}
24672467
}); err != nil {
@@ -2470,7 +2470,7 @@ func (m *kvMeta) cleanupLeakedInodesAndEdges(clean bool, before time.Duration) {
24702470
}
24712471

24722472
for inode, attr := range nodes {
2473-
if _, ok := foundInodes[inode]; !ok && time.Unix(attr.Ctime, 0).Before(cutoff) {
2473+
if _, ok := node2Edges[inode]; !ok && time.Unix(attr.Ctime, 0).Before(cutoff) {
24742474
logger.Infof("found leaded inode: %d %+v", inode, attr)
24752475
if clean {
24762476
err := m.doDeleteSustainedInode(0, inode)
@@ -2479,13 +2479,16 @@ func (m *kvMeta) cleanupLeakedInodesAndEdges(clean bool, before time.Duration) {
24792479
}
24802480
}
24812481
}
2482-
foundInodes[inode] = nil
2482+
node2Edges[inode] = nil
24832483
}
24842484

2485-
foundInodes[RootInode], foundInodes[TrashInode] = nil, nil
2485+
node2Edges[RootInode], node2Edges[TrashInode] = nil, nil
24862486
if err := m.client.txn(Background(), func(tx *kvTxn) error {
2487-
for c, e := range foundInodes {
2488-
if e != nil {
2487+
for c, es := range node2Edges {
2488+
if tx.get(m.inodeKey(c)) != nil {
2489+
continue
2490+
}
2491+
for _, e := range es {
24892492
logger.Infof("found leaked edge %d -> (%d, %s)", e.Parent, c, e.Name)
24902493
if clean {
24912494
tx.delete(m.entryKey(e.Parent, string(e.Name)))

0 commit comments

Comments
 (0)