From e9fda7b59e3a5dadca252c58302315aa625a34cf Mon Sep 17 00:00:00 2001 From: xuyuchao Date: Fri, 21 Nov 2025 12:03:41 +0800 Subject: [PATCH 1/2] meta/metrics: Add Prometheus metrics for trash and delayed slices --- pkg/meta/base.go | 76 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 67 insertions(+), 9 deletions(-) diff --git a/pkg/meta/base.go b/pkg/meta/base.go index 5e1904d7080c..3b04305c4712 100644 --- a/pkg/meta/base.go +++ b/pkg/meta/base.go @@ -268,15 +268,19 @@ type baseMeta struct { prefetchMu sync.Mutex prefetchedInodes freeID - usedSpaceG prometheus.Gauge - usedInodesG prometheus.Gauge - totalSpaceG prometheus.Gauge - totalInodesG prometheus.Gauge - txDist prometheus.Histogram - txRestart *prometheus.CounterVec - opDist prometheus.Histogram - opCount *prometheus.CounterVec - opDuration *prometheus.CounterVec + usedSpaceG prometheus.Gauge + usedInodesG prometheus.Gauge + totalSpaceG prometheus.Gauge + totalInodesG prometheus.Gauge + trashSpaceG prometheus.Gauge + trashInodesG prometheus.Gauge + delayedSlicesSpaceG prometheus.Gauge + delayedSlicesCountG prometheus.Gauge + txDist prometheus.Histogram + txRestart *prometheus.CounterVec + opDist prometheus.Histogram + opCount *prometheus.CounterVec + opDuration *prometheus.CounterVec en engine } @@ -322,6 +326,22 @@ func newBaseMeta(addr string, conf *Config) *baseMeta { Name: "total_inodes", Help: "Total number of inodes.", }), + trashSpaceG: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "trash_space", + Help: "Total space used by trash in bytes.", + }), + trashInodesG: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "trash_inodes", + Help: "Total number of inodes in trash.", + }), + delayedSlicesSpaceG: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "delayed_slices_space", + Help: "Total space used by delayed slices in bytes.", + }), + delayedSlicesCountG: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "delayed_slices_count", + Help: "Total number of delayed slices.", + }), txDist: prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "transaction_durations_histogram_seconds", Help: "Transactions latency distributions.", @@ -355,6 +375,10 @@ func (m *baseMeta) InitMetrics(reg prometheus.Registerer) { reg.MustRegister(m.usedInodesG) reg.MustRegister(m.totalSpaceG) reg.MustRegister(m.totalInodesG) + reg.MustRegister(m.trashSpaceG) + reg.MustRegister(m.trashInodesG) + reg.MustRegister(m.delayedSlicesSpaceG) + reg.MustRegister(m.delayedSlicesCountG) reg.MustRegister(m.txDist) reg.MustRegister(m.txRestart) reg.MustRegister(m.opDist) @@ -374,6 +398,8 @@ func (m *baseMeta) InitMetrics(reg prometheus.Registerer) { m.totalSpaceG.Set(float64(totalSpace)) m.totalInodesG.Set(float64(iused + iavail)) } + m.updateTrashMetrics(Background()) + m.updateDelayedSlicesMetrics(Background()) utils.SleepWithJitter(time.Second * 10) } }() @@ -386,6 +412,38 @@ func (m *baseMeta) timeit(method string, start time.Time) { m.opDuration.WithLabelValues(method).Add(used) } +func (m *baseMeta) updateTrashMetrics(ctx Context) { + var trashSpace int64 + var trashInodes uint64 + err := m.scanTrashEntry(ctx, func(_ Ino, length uint64) { + trashSpace += align4K(length) + trashInodes++ + }) + if err == nil { + if trashSpace < 0 { + trashSpace = 0 + } + m.trashSpaceG.Set(float64(trashSpace)) + m.trashInodesG.Set(float64(trashInodes)) + } +} + +func (m *baseMeta) updateDelayedSlicesMetrics(ctx Context) { + var delayedSlicesSpace uint64 + var delayedSlicesCount uint64 + err := m.en.scanTrashSlices(ctx, func(ss []Slice, _ int64) (bool, error) { + for _, s := range ss { + delayedSlicesSpace += uint64(s.Size) + delayedSlicesCount++ + } + return false, nil + }) + if err == nil { + m.delayedSlicesSpaceG.Set(float64(delayedSlicesSpace)) + m.delayedSlicesCountG.Set(float64(delayedSlicesCount)) + } +} + func (m *baseMeta) getBase() *baseMeta { return m } From 9dcd9640c170febf8960226c69ad4f7c45ac2598 Mon Sep 17 00:00:00 2001 From: xuyuchao Date: Fri, 21 Nov 2025 13:49:32 +0800 Subject: [PATCH 2/2] fix ut --- pkg/meta/base.go | 79 +++++++++++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 28 deletions(-) diff --git a/pkg/meta/base.go b/pkg/meta/base.go index 3b04305c4712..d9a0f81ae320 100644 --- a/pkg/meta/base.go +++ b/pkg/meta/base.go @@ -255,6 +255,12 @@ type baseMeta struct { fsStatsLock sync.Mutex *fsStat + trashMetricsLock sync.Mutex + trashSpace int64 + trashInodes uint64 + delayedSlicesSpace uint64 + delayedSlicesCount uint64 + parentMu sync.Mutex // protect dirParents quotaMu sync.RWMutex // protect dirQuotas dirParents map[Ino]Ino // directory inode -> parent inode @@ -385,6 +391,7 @@ func (m *baseMeta) InitMetrics(reg prometheus.Registerer) { reg.MustRegister(m.opCount) reg.MustRegister(m.opDuration) + go m.scanTrashAndDelayedSlicesMetrics() go func() { for { if m.sessCtx != nil && m.sessCtx.Canceled() { @@ -398,8 +405,13 @@ func (m *baseMeta) InitMetrics(reg prometheus.Registerer) { m.totalSpaceG.Set(float64(totalSpace)) m.totalInodesG.Set(float64(iused + iavail)) } - m.updateTrashMetrics(Background()) - m.updateDelayedSlicesMetrics(Background()) + m.trashMetricsLock.Lock() + m.trashSpaceG.Set(float64(m.trashSpace)) + m.trashInodesG.Set(float64(m.trashInodes)) + m.delayedSlicesSpaceG.Set(float64(m.delayedSlicesSpace)) + m.delayedSlicesCountG.Set(float64(m.delayedSlicesCount)) + m.trashMetricsLock.Unlock() + utils.SleepWithJitter(time.Second * 10) } }() @@ -412,35 +424,46 @@ func (m *baseMeta) timeit(method string, start time.Time) { m.opDuration.WithLabelValues(method).Add(used) } -func (m *baseMeta) updateTrashMetrics(ctx Context) { - var trashSpace int64 - var trashInodes uint64 - err := m.scanTrashEntry(ctx, func(_ Ino, length uint64) { - trashSpace += align4K(length) - trashInodes++ - }) - if err == nil { - if trashSpace < 0 { - trashSpace = 0 +func (m *baseMeta) scanTrashAndDelayedSlicesMetrics() { + for { + if m.sessCtx != nil && m.sessCtx.Canceled() { + return + } + ctx := Background() + + var trashSpace int64 + var trashInodes uint64 + err := m.scanTrashEntry(ctx, func(_ Ino, length uint64) { + trashSpace += align4K(length) + trashInodes++ + }) + if err == nil { + if trashSpace < 0 { + trashSpace = 0 + } + m.trashMetricsLock.Lock() + m.trashSpace = trashSpace + m.trashInodes = trashInodes + m.trashMetricsLock.Unlock() } - m.trashSpaceG.Set(float64(trashSpace)) - m.trashInodesG.Set(float64(trashInodes)) - } -} -func (m *baseMeta) updateDelayedSlicesMetrics(ctx Context) { - var delayedSlicesSpace uint64 - var delayedSlicesCount uint64 - err := m.en.scanTrashSlices(ctx, func(ss []Slice, _ int64) (bool, error) { - for _, s := range ss { - delayedSlicesSpace += uint64(s.Size) - delayedSlicesCount++ + var delayedSlicesSpace uint64 + var delayedSlicesCount uint64 + err = m.en.scanTrashSlices(ctx, func(ss []Slice, _ int64) (bool, error) { + for _, s := range ss { + delayedSlicesSpace += uint64(s.Size) + delayedSlicesCount++ + } + return false, nil + }) + if err == nil { + m.trashMetricsLock.Lock() + m.delayedSlicesSpace = delayedSlicesSpace + m.delayedSlicesCount = delayedSlicesCount + m.trashMetricsLock.Unlock() } - return false, nil - }) - if err == nil { - m.delayedSlicesSpaceG.Set(float64(delayedSlicesSpace)) - m.delayedSlicesCountG.Set(float64(delayedSlicesCount)) + + utils.SleepWithJitter(time.Second * 10) } }