From 3bacf168056e3591e386fc266a8ca3ea1db09217 Mon Sep 17 00:00:00 2001 From: xuyuchao Date: Mon, 10 Nov 2025 09:49:31 +0800 Subject: [PATCH 1/8] meta: optimizate the rmr emptyDir --- pkg/meta/base.go | 1 + pkg/meta/interface.go | 8 ++ pkg/meta/redis.go | 5 + pkg/meta/sql.go | 211 ++++++++++++++++++++++++++++++++++++++++++ pkg/meta/tkv.go | 5 + pkg/meta/utils.go | 56 +++++++---- 6 files changed, 268 insertions(+), 18 deletions(-) diff --git a/pkg/meta/base.go b/pkg/meta/base.go index 3210c144e241..372ff4ec80c9 100644 --- a/pkg/meta/base.go +++ b/pkg/meta/base.go @@ -115,6 +115,7 @@ type engine interface { doLink(ctx Context, inode, parent Ino, name string, attr *Attr) syscall.Errno doUnlink(ctx Context, parent Ino, name string, attr *Attr, skipCheckTrash ...bool) syscall.Errno doRmdir(ctx Context, parent Ino, name string, inode *Ino, attr *Attr, skipCheckTrash ...bool) syscall.Errno + doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *int64, space *int64, inodes *int64, userGroupQuotas *[]UserGroupQuotaDelta, skipCheckTrash ...bool) (errno syscall.Errno) doReadlink(ctx Context, inode Ino, noatime bool) (int64, []byte, error) doReaddir(ctx Context, inode Ino, plus uint8, entries *[]*Entry, limit int) syscall.Errno doRename(ctx Context, parentSrc Ino, nameSrc string, parentDst Ino, nameDst string, flags uint32, inode, tinode *Ino, attr, tattr *Attr) syscall.Errno diff --git a/pkg/meta/interface.go b/pkg/meta/interface.go index 2fc58eb0443c..d93bd4fc448e 100644 --- a/pkg/meta/interface.go +++ b/pkg/meta/interface.go @@ -327,6 +327,14 @@ type Summary struct { Dirs uint64 } +// UserGroupQuotaDelta represents quota changes for a specific user and group. +type UserGroupQuotaDelta struct { + Uid uint32 + Gid uint32 + Space int64 + Inodes int64 +} + type TreeSummary struct { Inode Ino Path string diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go index 335ab4d574b2..43a05d7f90b7 100644 --- a/pkg/meta/redis.go +++ b/pkg/meta/redis.go @@ -1666,6 +1666,11 @@ func (m *redisMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, s return errno(err) } +func (m *redisMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *int64, space *int64, inodes *int64, userGroupQuotas *[]UserGroupQuotaDelta, skipCheckTrash ...bool) syscall.Errno { + // redis 引擎不支持批量删除优化，返回不支持的错误 + return syscall.ENOTSUP +} + func (m *redisMeta) doRmdir(ctx Context, parent Ino, name string, pinode *Ino, oldAttr *Attr, skipCheckTrash ...bool) syscall.Errno { var trash Ino if !(len(skipCheckTrash) == 1 && skipCheckTrash[0]) { diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index cb6ca664b4d6..dbbbdbecf54f 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -2586,6 +2586,217 @@ func (m *dbMeta) doReaddir(ctx Context, inode Ino, plus uint8, entries *[]*Entry })) } +// recordDeletionStats 记录删除统计信息和用户组配额变化 +func recordDeletionStats( + n *node, + entrySpace int64, + spaceDelta int64, // 配额空间变化（0 表示空间不变，负数表示空间减少） + totalLength *int64, + totalSpace *int64, + totalInodes *int64, + userGroupQuotas *[]UserGroupQuotaDelta, + trash Ino, +) { + // 统计删除的空间和 inode + *totalLength += int64(n.Length) + *totalSpace += entrySpace + *totalInodes++ + + // 收集用户组配额变化，只在非回收站删除时收集 + if userGroupQuotas != nil && trash == 0 && n.Uid > 0 { + *userGroupQuotas = append(*userGroupQuotas, UserGroupQuotaDelta{ + Uid: n.Uid, + Gid: n.Gid, + Space: spaceDelta, + Inodes: -1, + }) + } +} + +func (m *dbMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *int64, space *int64, inodes *int64, userGroupQuotas *[]UserGroupQuotaDelta, skipCheckTrash ...bool) syscall.Errno { + if len(entries) == 0 { + return 0 + } + + var trash Ino + if len(skipCheckTrash) == 0 || !skipCheckTrash[0] { + if st := m.checkTrash(parent, &trash); st != 0 { + return st + } + } + + type entryInfo struct { + e edge + n node + opened bool + trash Ino + trashName string // 缓存回收站条目名称，避免重复计算 + } + + var entryInfos []entryInfo + var totalLength, totalSpace, totalInodes int64 + // 收集用户组配额变化，只在非回收站删除时收集 + if userGroupQuotas != nil { + *userGroupQuotas = make([]UserGroupQuotaDelta, 0, len(entries)) + } + + err := m.txn(func(s *xorm.Session) error { + pn := node{Inode: parent} + ok, err := s.Get(&pn) + if err != nil { + return err + } + if !ok { + return syscall.ENOENT + } + if pn.Type != TypeDirectory { + return syscall.ENOTDIR + } + if (pn.Flags&FlagAppend != 0) || (pn.Flags&FlagImmutable) != 0 { + return syscall.EPERM + } + + entryInfos = make([]entryInfo, 0, len(entries)) + now := time.Now().UnixNano() + + for _, entry := range entries { + e := edge{Parent: parent, Name: entry.Name, Inode: entry.Inode} + if entry.Attr != nil { + e.Type = entry.Attr.Typ + } + + info := entryInfo{e: e, trash: trash} + n := node{Inode: e.Inode} + ok, err := s.ForUpdate().Get(&n) + if err != nil { + return err + } + if !ok { + continue + } + if ctx.Uid() != 0 && pn.Mode&01000 != 0 && ctx.Uid() != pn.Uid && ctx.Uid() != n.Uid { + return syscall.EACCES + } + + if (n.Flags&FlagAppend) != 0 || (n.Flags&FlagImmutable) != 0 { + return syscall.EPERM + } + if (n.Flags & FlagSkipTrash) != 0 { + info.trash = 0 + } + + if info.trash > 0 { + info.trashName = m.trashEntry(parent, e.Inode, string(e.Name)) + if n.Nlink > 1 { + if o, err := s.Get(&edge{Parent: info.trash, Name: []byte(info.trashName), Inode: e.Inode, Type: e.Type}); err == nil && o { + info.trash = 0 + } + } + } + + n.setCtime(now) + if info.trash == 0 { + n.Nlink-- + if n.Type == TypeFile && n.Nlink == 0 && m.sid > 0 { + info.opened = m.of.IsOpen(e.Inode) + } + } else if n.Parent > 0 { + n.Parent = info.trash + } + + info.n = n + entryInfos = append(entryInfos, info) + } + + for _, info := range entryInfos { + if info.e.Type == TypeDirectory { + continue + } + if _, err := s.Delete(&edge{Parent: parent, Name: info.e.Name}); err != nil { + return err + } + + if info.n.Nlink > 0 { + if _, err := s.Cols("nlink", "ctime", "ctimensec", "parent").Update(&info.n, &node{Inode: info.e.Inode}); err != nil { + return err + } + if info.trash > 0 { + if err := mustInsert(s, &edge{Parent: info.trash, Name: []byte(info.trashName), Inode: info.e.Inode, Type: info.e.Type}); err != nil { + return err + } + } + entrySpace := align4K(info.n.Length) + recordDeletionStats(&info.n, entrySpace, 0, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) + } else { + switch info.e.Type { + case TypeFile: + entrySpace := align4K(info.n.Length) + if info.opened { + if err = mustInsert(s, sustained{Sid: m.sid, Inode: info.e.Inode}); err != nil { + return err + } + if _, err := s.Cols("nlink", "ctime", "ctimensec").Update(&info.n, &node{Inode: info.e.Inode}); err != nil { + return err + } + recordDeletionStats(&info.n, entrySpace, 0, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) + } else { + if err = mustInsert(s, delfile{info.e.Inode, info.n.Length, time.Now().Unix()}); err != nil { + return err + } + if _, err := s.Delete(&node{Inode: info.e.Inode}); err != nil { + return err + } + recordDeletionStats(&info.n, entrySpace, -entrySpace, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) + } + case TypeSymlink: + if _, err := s.Delete(&symlink{Inode: info.e.Inode}); err != nil { + return err + } + if _, err := s.Delete(&node{Inode: info.e.Inode}); err != nil { + return err + } + entrySpace := align4K(0) + recordDeletionStats(&info.n, entrySpace, -entrySpace, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) + default: + if _, err := s.Delete(&node{Inode: info.e.Inode}); err != nil { + return err + } + if info.e.Type != TypeFile { + entrySpace := align4K(0) + recordDeletionStats(&info.n, entrySpace, -entrySpace, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) + } + } + if _, err := s.Delete(&xattr{Inode: info.e.Inode}); err != nil { + return err + } + } + m.of.InvalidateChunk(info.e.Inode, invalidateAttrOnly) + } + + return nil + }) + + if err != nil { + return errno(err) + } + + // 事务外处理：fileDeleted 和 updateStats + if trash == 0 { + for _, info := range entryInfos { + if info.n.Type == TypeFile && info.n.Nlink == 0 { + isTrash := parent.IsTrash() + m.fileDeleted(info.opened, isTrash, info.e.Inode, info.n.Length) + } + } + m.updateStats(totalSpace, totalInodes) + } + + *length = totalLength + *space = totalSpace + *inodes = totalInodes + return 0 +} + func (m *dbMeta) doCleanStaleSession(sid uint64) error { var fail bool // release locks diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go index cbafbd6d924d..65df14aa30a0 100644 --- a/pkg/meta/tkv.go +++ b/pkg/meta/tkv.go @@ -1427,6 +1427,11 @@ func (m *kvMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, skip return errno(err) } +func (m *kvMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *int64, space *int64, inodes *int64, userGroupQuotas *[]UserGroupQuotaDelta, skipCheckTrash ...bool) syscall.Errno { + // tkv 引擎不支持批量删除优化，返回不支持的错误 + return syscall.ENOTSUP +} + func (m *kvMeta) doRmdir(ctx Context, parent Ino, name string, pinode *Ino, oldAttr *Attr, skipCheckTrash ...bool) syscall.Errno { var trash Ino if !(len(skipCheckTrash) == 1 && skipCheckTrash[0]) { diff --git a/pkg/meta/utils.go b/pkg/meta/utils.go index 833ed0c96275..76d39d9636e3 100644 --- a/pkg/meta/utils.go +++ b/pkg/meta/utils.go @@ -277,15 +277,7 @@ func (m *baseMeta) emptyDir(ctx Context, inode Ino, skipCheckTrash bool, count * } var wg sync.WaitGroup var status syscall.Errno - // try directories first to increase parallel - var dirs int - for i, e := range entries { - if e.Attr.Typ == TypeDirectory { - entries[dirs], entries[i] = entries[i], entries[dirs] - dirs++ - } - } - for i, e := range entries { + for _, e := range entries { if e.Attr.Typ == TypeDirectory { select { case concurrent <- 1: @@ -304,21 +296,49 @@ func (m *baseMeta) emptyDir(ctx Context, inode Ino, skipCheckTrash bool, count * return st } } - } else { - if count != nil { - atomic.AddUint64(count, 1) + } + if ctx.Canceled() { + return syscall.EINTR + } + + } + wg.Wait() + var nonDirEntries []*Entry + for _, e := range entries { + if e.Attr.Typ != TypeDirectory { + nonDirEntries = append(nonDirEntries, e) + } + } + var length int64 + var space int64 + var inodes int64 + var userGroupQuotas []UserGroupQuotaDelta + st := m.en.doEmptyDir(ctx, inode, nonDirEntries, &length, &space, &inodes, &userGroupQuotas, skipCheckTrash) + if st == 0 { + m.updateDirStat(ctx, inode, -length, -space, -inodes) + if !inode.IsTrash() { + m.updateDirQuota(ctx, inode, -space, -inodes) + for _, quota := range userGroupQuotas { + m.updateUserGroupQuota(ctx, quota.Uid, quota.Gid, quota.Space, quota.Inodes) + } + } + } else if st == syscall.ENOTSUP { + for _, e := range entries { + if e.Attr.Typ == TypeDirectory { + continue + } + if ctx.Canceled() { + return syscall.EINTR } if st := m.Unlink(ctx, inode, string(e.Name), skipCheckTrash); st != 0 && st != syscall.ENOENT { - ctx.Cancel() return st } } - if ctx.Canceled() { - return syscall.EINTR - } - entries[i] = nil // release memory + } else if st != 0 { + return st } - wg.Wait() + entries = nil + if status != 0 || inode == TrashInode { // try only once for .trash return status } From 6677229cc47788fb2e37e5de5731d2dfff84574a Mon Sep 17 00:00:00 2001 From: xuyuchao Date: Mon, 10 Nov 2025 21:59:55 +0800 Subject: [PATCH 2/8] fix ut --- pkg/meta/sql.go | 40 ++++++++++++++++++++++++++++------------ pkg/meta/utils.go | 14 ++++++-------- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index dbbbdbecf54f..8e6f0c366b7c 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -2630,12 +2630,12 @@ func (m *dbMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *i n node opened bool trash Ino - trashName string // 缓存回收站条目名称，避免重复计算 + trashName string + lastLink bool } var entryInfos []entryInfo var totalLength, totalSpace, totalInodes int64 - // 收集用户组配额变化，只在非回收站删除时收集 if userGroupQuotas != nil { *userGroupQuotas = make([]UserGroupQuotaDelta, 0, len(entries)) } @@ -2695,12 +2695,7 @@ func (m *dbMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *i } n.setCtime(now) - if info.trash == 0 { - n.Nlink-- - if n.Type == TypeFile && n.Nlink == 0 && m.sid > 0 { - info.opened = m.of.IsOpen(e.Inode) - } - } else if n.Parent > 0 { + if info.trash != 0 && n.Parent > 0 { n.Parent = info.trash } @@ -2708,6 +2703,27 @@ func (m *dbMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *i entryInfos = append(entryInfos, info) } + seen := make(map[Ino]int) + for i := range entryInfos { + info := &entryInfos[i] + if info.e.Type == TypeDirectory { + continue + } + original := int64(info.n.Nlink) + processed := seen[info.e.Inode] + finalNlink := original - int64(processed+1) + if finalNlink < 0 { + finalNlink = 0 + } + info.lastLink = (info.trash == 0 && finalNlink == 0) + if info.lastLink && info.e.Type == TypeFile && m.sid > 0 { + info.opened = m.of.IsOpen(info.e.Inode) + } + info.n.Nlink = uint32(finalNlink) + seen[info.e.Inode] = processed + 1 + } + + trashInserted := make(map[Ino]bool) for _, info := range entryInfos { if info.e.Type == TypeDirectory { continue @@ -2720,10 +2736,11 @@ func (m *dbMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *i if _, err := s.Cols("nlink", "ctime", "ctimensec", "parent").Update(&info.n, &node{Inode: info.e.Inode}); err != nil { return err } - if info.trash > 0 { + if info.trash > 0 && !trashInserted[info.e.Inode] { if err := mustInsert(s, &edge{Parent: info.trash, Name: []byte(info.trashName), Inode: info.e.Inode, Type: info.e.Type}); err != nil { return err } + trashInserted[info.e.Inode] = true } entrySpace := align4K(info.n.Length) recordDeletionStats(&info.n, entrySpace, 0, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) @@ -2780,15 +2797,14 @@ func (m *dbMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *i return errno(err) } - // 事务外处理：fileDeleted 和 updateStats if trash == 0 { for _, info := range entryInfos { - if info.n.Type == TypeFile && info.n.Nlink == 0 { + if info.n.Type == TypeFile && info.lastLink { isTrash := parent.IsTrash() m.fileDeleted(info.opened, isTrash, info.e.Inode, info.n.Length) } } - m.updateStats(totalSpace, totalInodes) + m.updateStats(-totalSpace, -totalInodes) } *length = totalLength diff --git a/pkg/meta/utils.go b/pkg/meta/utils.go index 76d39d9636e3..4fb8df686d66 100644 --- a/pkg/meta/utils.go +++ b/pkg/meta/utils.go @@ -277,6 +277,7 @@ func (m *baseMeta) emptyDir(ctx Context, inode Ino, skipCheckTrash bool, count * } var wg sync.WaitGroup var status syscall.Errno + var nonDirEntries []*Entry for _, e := range entries { if e.Attr.Typ == TypeDirectory { select { @@ -296,19 +297,16 @@ func (m *baseMeta) emptyDir(ctx Context, inode Ino, skipCheckTrash bool, count * return st } } - } + } else { + nonDirEntries = append(nonDirEntries, e) + } if ctx.Canceled() { return syscall.EINTR } } wg.Wait() - var nonDirEntries []*Entry - for _, e := range entries { - if e.Attr.Typ != TypeDirectory { - nonDirEntries = append(nonDirEntries, e) - } - } + var length int64 var space int64 var inodes int64 @@ -319,7 +317,7 @@ func (m *baseMeta) emptyDir(ctx Context, inode Ino, skipCheckTrash bool, count * if !inode.IsTrash() { m.updateDirQuota(ctx, inode, -space, -inodes) for _, quota := range userGroupQuotas { - m.updateUserGroupQuota(ctx, quota.Uid, quota.Gid, quota.Space, quota.Inodes) + m.updateUserGroupQuota(ctx, quota.Uid, quota.Gid, -quota.Space, -quota.Inodes) } } } else if st == syscall.ENOTSUP { From 37ce30bfde9667d98f04ec1c6cd79f906b41dece Mon Sep 17 00:00:00 2001 From: xuyuchao Date: Mon, 10 Nov 2025 22:37:50 +0800 Subject: [PATCH 3/8] fix ut --- pkg/meta/redis.go | 1 - pkg/meta/sql.go | 9 +++++---- pkg/meta/tkv.go | 1 - 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go index 43a05d7f90b7..4a26d581fcee 100644 --- a/pkg/meta/redis.go +++ b/pkg/meta/redis.go @@ -1667,7 +1667,6 @@ func (m *redisMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, s } func (m *redisMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *int64, space *int64, inodes *int64, userGroupQuotas *[]UserGroupQuotaDelta, skipCheckTrash ...bool) syscall.Errno { - // redis 引擎不支持批量删除优化，返回不支持的错误 return syscall.ENOTSUP } diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index 8e6f0c366b7c..25709fd1035c 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -2586,23 +2586,20 @@ func (m *dbMeta) doReaddir(ctx Context, inode Ino, plus uint8, entries *[]*Entry })) } -// recordDeletionStats 记录删除统计信息和用户组配额变化 func recordDeletionStats( n *node, entrySpace int64, - spaceDelta int64, // 配额空间变化（0 表示空间不变，负数表示空间减少） + spaceDelta int64, totalLength *int64, totalSpace *int64, totalInodes *int64, userGroupQuotas *[]UserGroupQuotaDelta, trash Ino, ) { - // 统计删除的空间和 inode *totalLength += int64(n.Length) *totalSpace += entrySpace *totalInodes++ - // 收集用户组配额变化，只在非回收站删除时收集 if userGroupQuotas != nil && trash == 0 && n.Uid > 0 { *userGroupQuotas = append(*userGroupQuotas, UserGroupQuotaDelta{ Uid: n.Uid, @@ -2715,6 +2712,10 @@ func (m *dbMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *i if finalNlink < 0 { finalNlink = 0 } + // If trash is enabled and this would be the last link, keep one link by moving it into trash. + if info.trash > 0 && finalNlink == 0 && info.e.Type != TypeDirectory { + finalNlink = 1 + } info.lastLink = (info.trash == 0 && finalNlink == 0) if info.lastLink && info.e.Type == TypeFile && m.sid > 0 { info.opened = m.of.IsOpen(info.e.Inode) diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go index 65df14aa30a0..0568a23cf8e3 100644 --- a/pkg/meta/tkv.go +++ b/pkg/meta/tkv.go @@ -1428,7 +1428,6 @@ func (m *kvMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, skip } func (m *kvMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *int64, space *int64, inodes *int64, userGroupQuotas *[]UserGroupQuotaDelta, skipCheckTrash ...bool) syscall.Errno { - // tkv 引擎不支持批量删除优化，返回不支持的错误 return syscall.ENOTSUP } From db929e5d4e3ed5aad0d3dd8462f40f92ba8da54f Mon Sep 17 00:00:00 2001 From: xuyuchao Date: Tue, 11 Nov 2025 16:45:31 +0800 Subject: [PATCH 4/8] Add count --- pkg/meta/utils.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/meta/utils.go b/pkg/meta/utils.go index 4fb8df686d66..8d8de771f329 100644 --- a/pkg/meta/utils.go +++ b/pkg/meta/utils.go @@ -320,6 +320,9 @@ func (m *baseMeta) emptyDir(ctx Context, inode Ino, skipCheckTrash bool, count * m.updateUserGroupQuota(ctx, quota.Uid, quota.Gid, -quota.Space, -quota.Inodes) } } + if count != nil && len(nonDirEntries) > 0 { + atomic.AddUint64(count, uint64(len(nonDirEntries))) + } } else if st == syscall.ENOTSUP { for _, e := range entries { if e.Attr.Typ == TypeDirectory { @@ -331,6 +334,9 @@ func (m *baseMeta) emptyDir(ctx Context, inode Ino, skipCheckTrash bool, count * if st := m.Unlink(ctx, inode, string(e.Name), skipCheckTrash); st != 0 && st != syscall.ENOENT { return st } + if count != nil { + atomic.AddUint64(count, 1) + } } } else if st != 0 { return st From 8676dc831c90588d647ba0a03260adb02f19716b Mon Sep 17 00:00:00 2001 From: xuyuchao Date: Fri, 14 Nov 2025 00:29:14 +0800 Subject: [PATCH 5/8] fix comments --- pkg/meta/base.go | 40 +++++++++++++- pkg/meta/redis.go | 2 +- pkg/meta/sql.go | 137 ++++++++++++++++++++++++++++++---------------- pkg/meta/tkv.go | 2 +- pkg/meta/utils.go | 44 ++------------- 5 files changed, 137 insertions(+), 88 deletions(-) diff --git a/pkg/meta/base.go b/pkg/meta/base.go index 372ff4ec80c9..866a6dc6ab08 100644 --- a/pkg/meta/base.go +++ b/pkg/meta/base.go @@ -115,7 +115,7 @@ type engine interface { doLink(ctx Context, inode, parent Ino, name string, attr *Attr) syscall.Errno doUnlink(ctx Context, parent Ino, name string, attr *Attr, skipCheckTrash ...bool) syscall.Errno doRmdir(ctx Context, parent Ino, name string, inode *Ino, attr *Attr, skipCheckTrash ...bool) syscall.Errno - doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *int64, space *int64, inodes *int64, userGroupQuotas *[]UserGroupQuotaDelta, skipCheckTrash ...bool) (errno syscall.Errno) + doBatchUnlink(ctx Context, parent Ino, entries []Entry, length *int64, space *int64, inodes *int64, userGroupQuotas *[]UserGroupQuotaDelta, skipCheckTrash ...bool) (errno syscall.Errno) doReadlink(ctx Context, inode Ino, noatime bool) (int64, []byte, error) doReaddir(ctx Context, inode Ino, plus uint8, entries *[]*Entry, limit int) syscall.Errno doRename(ctx Context, parentSrc Ino, nameSrc string, parentDst Ino, nameDst string, flags uint32, inode, tinode *Ino, attr, tattr *Attr) syscall.Errno @@ -1462,6 +1462,44 @@ func (m *baseMeta) Rmdir(ctx Context, parent Ino, name string, skipCheckTrash .. return st } +func (m *baseMeta) BatchUnlink(ctx Context, parent Ino, entries []Entry, count *uint64, skipCheckTrash bool) syscall.Errno { + var length int64 + var space int64 + var inodes int64 + var userGroupQuotas []UserGroupQuotaDelta + st := m.en.doBatchUnlink(ctx, parent, entries, &length, &space, &inodes, &userGroupQuotas, skipCheckTrash) + if st == 0 { + m.updateDirStat(ctx, parent, -length, -space, -inodes) + if !parent.IsTrash() { + m.updateDirQuota(ctx, parent, -space, -inodes) + for _, quota := range userGroupQuotas { + m.updateUserGroupQuota(ctx, quota.Uid, quota.Gid, -quota.Space, -quota.Inodes) + } + } + if count != nil && len(entries) > 0 { + atomic.AddUint64(count, uint64(len(entries))) + } + } else if st == syscall.ENOTSUP { + for _, e := range entries { + if e.Attr.Typ == TypeDirectory { + continue + } + if ctx.Canceled() { + return syscall.EINTR + } + if st := m.Unlink(ctx, parent, string(e.Name), skipCheckTrash); st != 0 && st != syscall.ENOENT { + return st + } + if count != nil { + atomic.AddUint64(count, 1) + } + } + } else if st != 0 { + return st + } + return 0 +} + func (m *baseMeta) Rename(ctx Context, parentSrc Ino, nameSrc string, parentDst Ino, nameDst string, flags uint32, inode *Ino, attr *Attr) syscall.Errno { if parentSrc == RootInode && nameSrc == TrashName || parentDst == RootInode && nameDst == TrashName { return syscall.EPERM diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go index 4a26d581fcee..0ba6a3ea108b 100644 --- a/pkg/meta/redis.go +++ b/pkg/meta/redis.go @@ -1666,7 +1666,7 @@ func (m *redisMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, s return errno(err) } -func (m *redisMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *int64, space *int64, inodes *int64, userGroupQuotas *[]UserGroupQuotaDelta, skipCheckTrash ...bool) syscall.Errno { +func (m *redisMeta) doBatchUnlink(ctx Context, parent Ino, entries []Entry, length *int64, space *int64, inodes *int64, userGroupQuotas *[]UserGroupQuotaDelta, skipCheckTrash ...bool) syscall.Errno { return syscall.ENOTSUP } diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index 25709fd1035c..f0fb3bf89fc8 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -2610,7 +2610,7 @@ func recordDeletionStats( } } -func (m *dbMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *int64, space *int64, inodes *int64, userGroupQuotas *[]UserGroupQuotaDelta, skipCheckTrash ...bool) syscall.Errno { +func (m *dbMeta) doBatchUnlink(ctx Context, parent Ino, entries []Entry, length *int64, space *int64, inodes *int64, userGroupQuotas *[]UserGroupQuotaDelta, skipCheckTrash ...bool) syscall.Errno { if len(entries) == 0 { return 0 } @@ -2630,13 +2630,11 @@ func (m *dbMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *i trashName string lastLink bool } - var entryInfos []entryInfo var totalLength, totalSpace, totalInodes int64 if userGroupQuotas != nil { *userGroupQuotas = make([]UserGroupQuotaDelta, 0, len(entries)) } - err := m.txn(func(s *xorm.Session) error { pn := node{Inode: parent} ok, err := s.Get(&pn) @@ -2656,64 +2654,96 @@ func (m *dbMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *i entryInfos = make([]entryInfo, 0, len(entries)) now := time.Now().UnixNano() - for _, entry := range entries { + inodes := make([]Ino, 0, len(entries)) + inodeToEntry := make(map[Ino][]int) // inode -> indices in entries + for i, entry := range entries { e := edge{Parent: parent, Name: entry.Name, Inode: entry.Inode} if entry.Attr != nil { e.Type = entry.Attr.Typ } - info := entryInfo{e: e, trash: trash} - n := node{Inode: e.Inode} - ok, err := s.ForUpdate().Get(&n) - if err != nil { - return err + entryInfos = append(entryInfos, info) + if _, exists := inodeToEntry[entry.Inode]; !exists { + inodes = append(inodes, entry.Inode) } - if !ok { - continue + inodeToEntry[entry.Inode] = append(inodeToEntry[entry.Inode], i) + } + + if len(inodes) > 0 { + var nodes []node + if err := s.ForUpdate().In("inode", inodes).Find(&nodes); err != nil { + return err } - if ctx.Uid() != 0 && pn.Mode&01000 != 0 && ctx.Uid() != pn.Uid && ctx.Uid() != n.Uid { - return syscall.EACCES + nodeMap := make(map[Ino]*node, len(nodes)) + for i := range nodes { + nodeMap[nodes[i].Inode] = &nodes[i] } - if (n.Flags&FlagAppend) != 0 || (n.Flags&FlagImmutable) != 0 { - return syscall.EPERM - } - if (n.Flags & FlagSkipTrash) != 0 { - info.trash = 0 + for i := range entryInfos { + info := &entryInfos[i] + n, ok := nodeMap[info.e.Inode] + if !ok { + entryInfos[i].e.Inode = 0 + continue + } + if ctx.Uid() != 0 && pn.Mode&01000 != 0 && ctx.Uid() != pn.Uid && ctx.Uid() != n.Uid { + return syscall.EACCES + } + if (n.Flags&FlagAppend) != 0 || (n.Flags&FlagImmutable) != 0 { + return syscall.EPERM + } + if (n.Flags & FlagSkipTrash) != 0 { + info.trash = 0 + } + info.n = *n } - if info.trash > 0 { - info.trashName = m.trashEntry(parent, e.Inode, string(e.Name)) - if n.Nlink > 1 { - if o, err := s.Get(&edge{Parent: info.trash, Name: []byte(info.trashName), Inode: e.Inode, Type: e.Type}); err == nil && o { - info.trash = 0 - } + filteredInfos := entryInfos[:0] + for i := range entryInfos { + if entryInfos[i].e.Inode != 0 { + filteredInfos = append(filteredInfos, entryInfos[i]) } } + entryInfos = filteredInfos + } - n.setCtime(now) - if info.trash != 0 && n.Parent > 0 { - n.Parent = info.trash + for i := range entryInfos { + info := &entryInfos[i] + if info.trash > 0 && info.n.Nlink > 1 { + info.trashName = m.trashEntry(parent, info.e.Inode, string(info.e.Name)) + te := edge{ + Parent: info.trash, + Name: []byte(info.trashName), + Inode: info.e.Inode, + Type: info.e.Type, + } + if ok, err := s.Get(&te); err == nil && ok { + info.trash = 0 + } } + } - info.n = n - entryInfos = append(entryInfos, info) + for i := range entryInfos { + info := &entryInfos[i] + info.n.setCtime(now) + if info.trash != 0 && info.n.Parent > 0 { + info.n.Parent = info.trash + } } - seen := make(map[Ino]int) + seen := make(map[Ino]uint32) for i := range entryInfos { info := &entryInfos[i] if info.e.Type == TypeDirectory { continue } - original := int64(info.n.Nlink) - processed := seen[info.e.Inode] - finalNlink := original - int64(processed+1) + processed := seen[info.e.Inode] + 1 + finalNlink := int64(info.n.Nlink) - int64(processed) if finalNlink < 0 { finalNlink = 0 } // If trash is enabled and this would be the last link, keep one link by moving it into trash. - if info.trash > 0 && finalNlink == 0 && info.e.Type != TypeDirectory { + if info.trash > 0 && finalNlink == 0 { finalNlink = 1 } info.lastLink = (info.trash == 0 && finalNlink == 0) @@ -2721,24 +2751,36 @@ func (m *dbMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *i info.opened = m.of.IsOpen(info.e.Inode) } info.n.Nlink = uint32(finalNlink) - seen[info.e.Inode] = processed + 1 + seen[info.e.Inode] = processed } trashInserted := make(map[Ino]bool) + nowUnix := time.Now().Unix() + for _, info := range entryInfos { if info.e.Type == TypeDirectory { continue } - if _, err := s.Delete(&edge{Parent: parent, Name: info.e.Name}); err != nil { + e := edge{Parent: parent, Name: info.e.Name} + if _, err := s.Delete(&e); err != nil { return err } if info.n.Nlink > 0 { - if _, err := s.Cols("nlink", "ctime", "ctimensec", "parent").Update(&info.n, &node{Inode: info.e.Inode}); err != nil { + if _, err := s.Cols("nlink", "ctime", "ctimensec", "parent").Update(&info.n, &node{Inode: info.n.Inode}); err != nil { return err } if info.trash > 0 && !trashInserted[info.e.Inode] { - if err := mustInsert(s, &edge{Parent: info.trash, Name: []byte(info.trashName), Inode: info.e.Inode, Type: info.e.Type}); err != nil { + if info.trashName == "" { + info.trashName = m.trashEntry(parent, info.e.Inode, string(info.e.Name)) + } + te := edge{ + Parent: info.trash, + Name: []byte(info.trashName), + Inode: info.e.Inode, + Type: info.e.Type, + } + if err := mustInsert(s, &te); err != nil { return err } trashInserted[info.e.Inode] = true @@ -2750,15 +2792,15 @@ func (m *dbMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *i case TypeFile: entrySpace := align4K(info.n.Length) if info.opened { - if err = mustInsert(s, sustained{Sid: m.sid, Inode: info.e.Inode}); err != nil { + if err := mustInsert(s, &sustained{Sid: m.sid, Inode: info.e.Inode}); err != nil { return err } - if _, err := s.Cols("nlink", "ctime", "ctimensec").Update(&info.n, &node{Inode: info.e.Inode}); err != nil { + if _, err := s.Cols("nlink", "ctime", "ctimensec").Update(&info.n, &node{Inode: info.n.Inode}); err != nil { return err } recordDeletionStats(&info.n, entrySpace, 0, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) } else { - if err = mustInsert(s, delfile{info.e.Inode, info.n.Length, time.Now().Unix()}); err != nil { + if err := mustInsert(s, &delfile{info.e.Inode, info.n.Length, nowUnix}); err != nil { return err } if _, err := s.Delete(&node{Inode: info.e.Inode}); err != nil { @@ -2770,11 +2812,7 @@ func (m *dbMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *i if _, err := s.Delete(&symlink{Inode: info.e.Inode}); err != nil { return err } - if _, err := s.Delete(&node{Inode: info.e.Inode}); err != nil { - return err - } - entrySpace := align4K(0) - recordDeletionStats(&info.n, entrySpace, -entrySpace, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) + fallthrough default: if _, err := s.Delete(&node{Inode: info.e.Inode}); err != nil { return err @@ -2788,7 +2826,12 @@ func (m *dbMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *i return err } } - m.of.InvalidateChunk(info.e.Inode, invalidateAttrOnly) + } + + for _, info := range entryInfos { + if info.e.Type != TypeDirectory { + m.of.InvalidateChunk(info.e.Inode, invalidateAttrOnly) + } } return nil diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go index 0568a23cf8e3..c3b80efcc442 100644 --- a/pkg/meta/tkv.go +++ b/pkg/meta/tkv.go @@ -1427,7 +1427,7 @@ func (m *kvMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, skip return errno(err) } -func (m *kvMeta) doEmptyDir(ctx Context, parent Ino, entries []*Entry, length *int64, space *int64, inodes *int64, userGroupQuotas *[]UserGroupQuotaDelta, skipCheckTrash ...bool) syscall.Errno { +func (m *kvMeta) doBatchUnlink(ctx Context, parent Ino, entries []Entry, length *int64, space *int64, inodes *int64, userGroupQuotas *[]UserGroupQuotaDelta, skipCheckTrash ...bool) syscall.Errno { return syscall.ENOTSUP } diff --git a/pkg/meta/utils.go b/pkg/meta/utils.go index 8d8de771f329..fc717e0233d5 100644 --- a/pkg/meta/utils.go +++ b/pkg/meta/utils.go @@ -277,8 +277,8 @@ func (m *baseMeta) emptyDir(ctx Context, inode Ino, skipCheckTrash bool, count * } var wg sync.WaitGroup var status syscall.Errno - var nonDirEntries []*Entry - for _, e := range entries { + var nonDirEntries []Entry + for i, e := range entries { if e.Attr.Typ == TypeDirectory { select { case concurrent <- 1: @@ -298,50 +298,18 @@ func (m *baseMeta) emptyDir(ctx Context, inode Ino, skipCheckTrash bool, count * } } } else { - nonDirEntries = append(nonDirEntries, e) + nonDirEntries = append(nonDirEntries, *e) } if ctx.Canceled() { return syscall.EINTR } - + entries[i] = nil // release memory } wg.Wait() - var length int64 - var space int64 - var inodes int64 - var userGroupQuotas []UserGroupQuotaDelta - st := m.en.doEmptyDir(ctx, inode, nonDirEntries, &length, &space, &inodes, &userGroupQuotas, skipCheckTrash) - if st == 0 { - m.updateDirStat(ctx, inode, -length, -space, -inodes) - if !inode.IsTrash() { - m.updateDirQuota(ctx, inode, -space, -inodes) - for _, quota := range userGroupQuotas { - m.updateUserGroupQuota(ctx, quota.Uid, quota.Gid, -quota.Space, -quota.Inodes) - } - } - if count != nil && len(nonDirEntries) > 0 { - atomic.AddUint64(count, uint64(len(nonDirEntries))) - } - } else if st == syscall.ENOTSUP { - for _, e := range entries { - if e.Attr.Typ == TypeDirectory { - continue - } - if ctx.Canceled() { - return syscall.EINTR - } - if st := m.Unlink(ctx, inode, string(e.Name), skipCheckTrash); st != 0 && st != syscall.ENOENT { - return st - } - if count != nil { - atomic.AddUint64(count, 1) - } - } - } else if st != 0 { - return st + if status == 0 { + status = m.BatchUnlink(ctx, inode, nonDirEntries, count, skipCheckTrash) } - entries = nil if status != 0 || inode == TrashInode { // try only once for .trash return status From 0937f4f4bebd57cafcdf9d6924190fb2cd561122 Mon Sep 17 00:00:00 2001 From: xuyuchao Date: Mon, 17 Nov 2025 12:00:15 +0800 Subject: [PATCH 6/8] Merge SQL statements --- pkg/meta/sql.go | 174 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 125 insertions(+), 49 deletions(-) diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index f0fb3bf89fc8..9fe73935be62 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -2698,27 +2698,52 @@ func (m *dbMeta) doBatchUnlink(ctx Context, parent Ino, entries []Entry, length info.n = *n } - filteredInfos := entryInfos[:0] + filtered := entryInfos[:0] for i := range entryInfos { if entryInfos[i].e.Inode != 0 { - filteredInfos = append(filteredInfos, entryInfos[i]) + filtered = append(filtered, entryInfos[i]) } } - entryInfos = filteredInfos + entryInfos = filtered } + // Batch check if trash entries already exist to avoid duplicate hardlinks + var edges []edge + var idxs []int for i := range entryInfos { info := &entryInfos[i] if info.trash > 0 && info.n.Nlink > 1 { info.trashName = m.trashEntry(parent, info.e.Inode, string(info.e.Name)) - te := edge{ + edges = append(edges, edge{ Parent: info.trash, Name: []byte(info.trashName), Inode: info.e.Inode, Type: info.e.Type, + }) + idxs = append(idxs, i) + } + } + if len(edges) > 0 { + var found []edge + q := s.Table(&edge{}) + for i, e := range edges { + if i == 0 { + q = q.Where("(parent = ? AND name = ?)", e.Parent, e.Name) + } else { + q = q.Or("(parent = ? AND name = ?)", e.Parent, e.Name) } - if ok, err := s.Get(&te); err == nil && ok { - info.trash = 0 + } + if err := q.Find(&found); err != nil { + return err + } + exists := make(map[string]bool, len(found)) + for _, e := range found { + exists[fmt.Sprintf("%d:%s", e.Parent, string(e.Name))] = true + } + for i, idx := range idxs { + key := fmt.Sprintf("%d:%s", edges[i].Parent, string(edges[i].Name)) + if exists[key] { + entryInfos[idx].trash = 0 } } } @@ -2754,78 +2779,129 @@ func (m *dbMeta) doBatchUnlink(ctx Context, parent Ino, entries []Entry, length seen[info.e.Inode] = processed } - trashInserted := make(map[Ino]bool) - nowUnix := time.Now().Unix() + trashMap := make(map[Ino]bool) + now = time.Now().Unix() + + var delEdges []edge + var updNodes []node + var trashEdges []edge + var sustaineds []sustained + var delfiles []delfile + var delNodes, delSymlinks, delXattrs []Ino for _, info := range entryInfos { if info.e.Type == TypeDirectory { continue } - e := edge{Parent: parent, Name: info.e.Name} - if _, err := s.Delete(&e); err != nil { - return err - } + delEdges = append(delEdges, edge{Parent: parent, Name: info.e.Name}) if info.n.Nlink > 0 { - if _, err := s.Cols("nlink", "ctime", "ctimensec", "parent").Update(&info.n, &node{Inode: info.n.Inode}); err != nil { - return err - } - if info.trash > 0 && !trashInserted[info.e.Inode] { + updNodes = append(updNodes, info.n) + if info.trash > 0 && !trashMap[info.e.Inode] { if info.trashName == "" { info.trashName = m.trashEntry(parent, info.e.Inode, string(info.e.Name)) } - te := edge{ + trashEdges = append(trashEdges, edge{ Parent: info.trash, Name: []byte(info.trashName), Inode: info.e.Inode, Type: info.e.Type, - } - if err := mustInsert(s, &te); err != nil { - return err - } - trashInserted[info.e.Inode] = true + }) + trashMap[info.e.Inode] = true } - entrySpace := align4K(info.n.Length) - recordDeletionStats(&info.n, entrySpace, 0, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) + recordDeletionStats(&info.n, align4K(info.n.Length), 0, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) } else { switch info.e.Type { case TypeFile: - entrySpace := align4K(info.n.Length) + space := align4K(info.n.Length) if info.opened { - if err := mustInsert(s, &sustained{Sid: m.sid, Inode: info.e.Inode}); err != nil { - return err - } - if _, err := s.Cols("nlink", "ctime", "ctimensec").Update(&info.n, &node{Inode: info.n.Inode}); err != nil { - return err - } - recordDeletionStats(&info.n, entrySpace, 0, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) + sustaineds = append(sustaineds, sustained{Sid: m.sid, Inode: info.e.Inode}) + updNodes = append(updNodes, info.n) + recordDeletionStats(&info.n, space, 0, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) } else { - if err := mustInsert(s, &delfile{info.e.Inode, info.n.Length, nowUnix}); err != nil { - return err - } - if _, err := s.Delete(&node{Inode: info.e.Inode}); err != nil { - return err - } - recordDeletionStats(&info.n, entrySpace, -entrySpace, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) + delfiles = append(delfiles, delfile{info.e.Inode, info.n.Length, now}) + delNodes = append(delNodes, info.e.Inode) + recordDeletionStats(&info.n, space, -space, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) } case TypeSymlink: - if _, err := s.Delete(&symlink{Inode: info.e.Inode}); err != nil { - return err - } + delSymlinks = append(delSymlinks, info.e.Inode) fallthrough default: - if _, err := s.Delete(&node{Inode: info.e.Inode}); err != nil { - return err - } + delNodes = append(delNodes, info.e.Inode) if info.e.Type != TypeFile { - entrySpace := align4K(0) - recordDeletionStats(&info.n, entrySpace, -entrySpace, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) + recordDeletionStats(&info.n, 0, 0, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) } } - if _, err := s.Delete(&xattr{Inode: info.e.Inode}); err != nil { - return err + delXattrs = append(delXattrs, info.e.Inode) + } + } + + // Batch delete edges + if len(delEdges) > 0 { + q := s.Table(&edge{}) + for i, e := range delEdges { + if i == 0 { + q = q.Where("(parent = ? AND name = ?)", e.Parent, e.Name) + } else { + q = q.Or("(parent = ? AND name = ?)", e.Parent, e.Name) } } + if _, err := q.Delete(&edge{}); err != nil { + return err + } + } + + // Batch update nodes + for _, n := range updNodes { + if _, err := s.Cols("nlink", "ctime", "ctimensec", "parent").Update(&n, &node{Inode: n.Inode}); err != nil { + return err + } + } + + // Batch inserts + if len(trashEdges) > 0 { + items := make([]interface{}, len(trashEdges)) + for i := range trashEdges { + items[i] = &trashEdges[i] + } + if err := mustInsert(s, items...); err != nil { + return err + } + } + if len(sustaineds) > 0 { + items := make([]interface{}, len(sustaineds)) + for i := range sustaineds { + items[i] = &sustaineds[i] + } + if err := mustInsert(s, items...); err != nil { + return err + } + } + if len(delfiles) > 0 { + items := make([]interface{}, len(delfiles)) + for i := range delfiles { + items[i] = &delfiles[i] + } + if err := mustInsert(s, items...); err != nil { + return err + } + } + + // Batch deletes + if len(delNodes) > 0 { + if _, err := s.In("inode", delNodes).Delete(&node{}); err != nil { + return err + } + } + if len(delSymlinks) > 0 { + if _, err := s.In("inode", delSymlinks).Delete(&symlink{}); err != nil { + return err + } + } + if len(delXattrs) > 0 { + if _, err := s.In("inode", delXattrs).Delete(&xattr{}); err != nil { + return err + } } for _, info := range entryInfos { From 16d439a75ecbee3dd61593ac67c78030ac7804c2 Mon Sep 17 00:00:00 2001 From: jiefenghuang Date: Tue, 18 Nov 2025 12:21:42 +0800 Subject: [PATCH 7/8] review Signed-off-by: jiefenghuang --- pkg/meta/sql.go | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index 9fe73935be62..5a9497fc554f 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -2589,7 +2589,7 @@ func (m *dbMeta) doReaddir(ctx Context, inode Ino, plus uint8, entries *[]*Entry func recordDeletionStats( n *node, entrySpace int64, - spaceDelta int64, + spaceDelta int64, totalLength *int64, totalSpace *int64, totalInodes *int64, @@ -2623,12 +2623,12 @@ func (m *dbMeta) doBatchUnlink(ctx Context, parent Ino, entries []Entry, length } type entryInfo struct { - e edge - n node - opened bool - trash Ino - trashName string - lastLink bool + e edge + n node + opened bool + trash Ino + trashName string + lastLink bool } var entryInfos []entryInfo var totalLength, totalSpace, totalInodes int64 @@ -2647,6 +2647,11 @@ func (m *dbMeta) doBatchUnlink(ctx Context, parent Ino, entries []Entry, length if pn.Type != TypeDirectory { return syscall.ENOTDIR } + var pattr Attr + m.parseAttr(&pn, &pattr) + if st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 { + return st + } if (pn.Flags&FlagAppend != 0) || (pn.Flags&FlagImmutable) != 0 { return syscall.EPERM } @@ -2655,18 +2660,17 @@ func (m *dbMeta) doBatchUnlink(ctx Context, parent Ino, entries []Entry, length now := time.Now().UnixNano() inodes := make([]Ino, 0, len(entries)) - inodeToEntry := make(map[Ino][]int) // inode -> indices in entries - for i, entry := range entries { + inodeM := make(map[Ino]struct{}) // filter hardlinks + for _, entry := range entries { e := edge{Parent: parent, Name: entry.Name, Inode: entry.Inode} if entry.Attr != nil { e.Type = entry.Attr.Typ } - info := entryInfo{e: e, trash: trash} - entryInfos = append(entryInfos, info) - if _, exists := inodeToEntry[entry.Inode]; !exists { + entryInfos = append(entryInfos, entryInfo{e: e, trash: trash}) + if _, exists := inodeM[entry.Inode]; !exists { + inodeM[entry.Inode] = struct{}{} inodes = append(inodes, entry.Inode) } - inodeToEntry[entry.Inode] = append(inodeToEntry[entry.Inode], i) } if len(inodes) > 0 { @@ -2674,6 +2678,7 @@ func (m *dbMeta) doBatchUnlink(ctx Context, parent Ino, entries []Entry, length if err := s.ForUpdate().In("inode", inodes).Find(&nodes); err != nil { return err } + // some inodes may not exist nodeMap := make(map[Ino]*node, len(nodes)) for i := range nodes { nodeMap[nodes[i].Inode] = &nodes[i] @@ -2919,7 +2924,7 @@ func (m *dbMeta) doBatchUnlink(ctx Context, parent Ino, entries []Entry, length if trash == 0 { for _, info := range entryInfos { - if info.n.Type == TypeFile && info.lastLink { + if info.n.Type == TypeFile && info.lastLink { isTrash := parent.IsTrash() m.fileDeleted(info.opened, isTrash, info.e.Inode, info.n.Length) } From baf6b28a2dcc53f4dde0eec9eae2837e58fcc801 Mon Sep 17 00:00:00 2001 From: xuyuchao Date: Wed, 19 Nov 2025 11:04:27 +0800 Subject: [PATCH 8/8] fix bug --- pkg/meta/sql.go | 189 +++++++++++++++--------------------------------- 1 file changed, 57 insertions(+), 132 deletions(-) diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index 5a9497fc554f..b0a6076ee420 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -2703,52 +2703,27 @@ func (m *dbMeta) doBatchUnlink(ctx Context, parent Ino, entries []Entry, length info.n = *n } - filtered := entryInfos[:0] + filteredInfos := entryInfos[:0] for i := range entryInfos { if entryInfos[i].e.Inode != 0 { - filtered = append(filtered, entryInfos[i]) + filteredInfos = append(filteredInfos, entryInfos[i]) } } - entryInfos = filtered + entryInfos = filteredInfos } - // Batch check if trash entries already exist to avoid duplicate hardlinks - var edges []edge - var idxs []int for i := range entryInfos { info := &entryInfos[i] if info.trash > 0 && info.n.Nlink > 1 { info.trashName = m.trashEntry(parent, info.e.Inode, string(info.e.Name)) - edges = append(edges, edge{ + te := edge{ Parent: info.trash, Name: []byte(info.trashName), Inode: info.e.Inode, Type: info.e.Type, - }) - idxs = append(idxs, i) - } - } - if len(edges) > 0 { - var found []edge - q := s.Table(&edge{}) - for i, e := range edges { - if i == 0 { - q = q.Where("(parent = ? AND name = ?)", e.Parent, e.Name) - } else { - q = q.Or("(parent = ? AND name = ?)", e.Parent, e.Name) } - } - if err := q.Find(&found); err != nil { - return err - } - exists := make(map[string]bool, len(found)) - for _, e := range found { - exists[fmt.Sprintf("%d:%s", e.Parent, string(e.Name))] = true - } - for i, idx := range idxs { - key := fmt.Sprintf("%d:%s", edges[i].Parent, string(edges[i].Name)) - if exists[key] { - entryInfos[idx].trash = 0 + if ok, err := s.Get(&te); err == nil && ok { + info.trash = 0 } } } @@ -2768,13 +2743,14 @@ func (m *dbMeta) doBatchUnlink(ctx Context, parent Ino, entries []Entry, length continue } processed := seen[info.e.Inode] + 1 - finalNlink := int64(info.n.Nlink) - int64(processed) - if finalNlink < 0 { - finalNlink = 0 - } - // If trash is enabled and this would be the last link, keep one link by moving it into trash. - if info.trash > 0 && finalNlink == 0 { - finalNlink = 1 + var finalNlink int64 + if info.trash == 0 { + finalNlink = int64(info.n.Nlink) - int64(processed) + if finalNlink < 0 { + finalNlink = 0 + } + } else { + finalNlink = int64(info.n.Nlink) } info.lastLink = (info.trash == 0 && finalNlink == 0) if info.lastLink && info.e.Type == TypeFile && m.sid > 0 { @@ -2784,129 +2760,78 @@ func (m *dbMeta) doBatchUnlink(ctx Context, parent Ino, entries []Entry, length seen[info.e.Inode] = processed } - trashMap := make(map[Ino]bool) - now = time.Now().Unix() - - var delEdges []edge - var updNodes []node - var trashEdges []edge - var sustaineds []sustained - var delfiles []delfile - var delNodes, delSymlinks, delXattrs []Ino + trashInserted := make(map[Ino]bool) + nowUnix := time.Now().Unix() for _, info := range entryInfos { if info.e.Type == TypeDirectory { continue } - delEdges = append(delEdges, edge{Parent: parent, Name: info.e.Name}) + e := edge{Parent: parent, Name: info.e.Name} + if _, err := s.Delete(&e); err != nil { + return err + } if info.n.Nlink > 0 { - updNodes = append(updNodes, info.n) - if info.trash > 0 && !trashMap[info.e.Inode] { + if _, err := s.Cols("nlink", "ctime", "ctimensec", "parent").Update(&info.n, &node{Inode: info.n.Inode}); err != nil { + return err + } + if info.trash > 0 && !trashInserted[info.e.Inode] { if info.trashName == "" { info.trashName = m.trashEntry(parent, info.e.Inode, string(info.e.Name)) } - trashEdges = append(trashEdges, edge{ + te := edge{ Parent: info.trash, Name: []byte(info.trashName), Inode: info.e.Inode, Type: info.e.Type, - }) - trashMap[info.e.Inode] = true + } + if err := mustInsert(s, &te); err != nil { + return err + } + trashInserted[info.e.Inode] = true } - recordDeletionStats(&info.n, align4K(info.n.Length), 0, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) + entrySpace := align4K(info.n.Length) + recordDeletionStats(&info.n, entrySpace, 0, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) } else { switch info.e.Type { case TypeFile: - space := align4K(info.n.Length) + entrySpace := align4K(info.n.Length) if info.opened { - sustaineds = append(sustaineds, sustained{Sid: m.sid, Inode: info.e.Inode}) - updNodes = append(updNodes, info.n) - recordDeletionStats(&info.n, space, 0, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) + if err := mustInsert(s, &sustained{Sid: m.sid, Inode: info.e.Inode}); err != nil { + return err + } + if _, err := s.Cols("nlink", "ctime", "ctimensec").Update(&info.n, &node{Inode: info.n.Inode}); err != nil { + return err + } + recordDeletionStats(&info.n, entrySpace, 0, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) } else { - delfiles = append(delfiles, delfile{info.e.Inode, info.n.Length, now}) - delNodes = append(delNodes, info.e.Inode) - recordDeletionStats(&info.n, space, -space, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) + if err := mustInsert(s, &delfile{info.e.Inode, info.n.Length, nowUnix}); err != nil { + return err + } + if _, err := s.Delete(&node{Inode: info.e.Inode}); err != nil { + return err + } + recordDeletionStats(&info.n, entrySpace, -entrySpace, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) } case TypeSymlink: - delSymlinks = append(delSymlinks, info.e.Inode) + if _, err := s.Delete(&symlink{Inode: info.e.Inode}); err != nil { + return err + } fallthrough default: - delNodes = append(delNodes, info.e.Inode) + if _, err := s.Delete(&node{Inode: info.e.Inode}); err != nil { + return err + } if info.e.Type != TypeFile { - recordDeletionStats(&info.n, 0, 0, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) + entrySpace := align4K(0) + recordDeletionStats(&info.n, entrySpace, -entrySpace, &totalLength, &totalSpace, &totalInodes, userGroupQuotas, trash) } } - delXattrs = append(delXattrs, info.e.Inode) - } - } - - // Batch delete edges - if len(delEdges) > 0 { - q := s.Table(&edge{}) - for i, e := range delEdges { - if i == 0 { - q = q.Where("(parent = ? AND name = ?)", e.Parent, e.Name) - } else { - q = q.Or("(parent = ? AND name = ?)", e.Parent, e.Name) + if _, err := s.Delete(&xattr{Inode: info.e.Inode}); err != nil { + return err } } - if _, err := q.Delete(&edge{}); err != nil { - return err - } - } - - // Batch update nodes - for _, n := range updNodes { - if _, err := s.Cols("nlink", "ctime", "ctimensec", "parent").Update(&n, &node{Inode: n.Inode}); err != nil { - return err - } - } - - // Batch inserts - if len(trashEdges) > 0 { - items := make([]interface{}, len(trashEdges)) - for i := range trashEdges { - items[i] = &trashEdges[i] - } - if err := mustInsert(s, items...); err != nil { - return err - } - } - if len(sustaineds) > 0 { - items := make([]interface{}, len(sustaineds)) - for i := range sustaineds { - items[i] = &sustaineds[i] - } - if err := mustInsert(s, items...); err != nil { - return err - } - } - if len(delfiles) > 0 { - items := make([]interface{}, len(delfiles)) - for i := range delfiles { - items[i] = &delfiles[i] - } - if err := mustInsert(s, items...); err != nil { - return err - } - } - - // Batch deletes - if len(delNodes) > 0 { - if _, err := s.In("inode", delNodes).Delete(&node{}); err != nil { - return err - } - } - if len(delSymlinks) > 0 { - if _, err := s.In("inode", delSymlinks).Delete(&symlink{}); err != nil { - return err - } - } - if len(delXattrs) > 0 { - if _, err := s.In("inode", delXattrs).Delete(&xattr{}); err != nil { - return err - } } for _, info := range entryInfos {