meta/metrics: Add Prometheus metrics for trash and delayed slices

pkg/meta/base.go

@@ -255,6 +255,12 @@ type baseMeta struct {
 	fsStatsLock sync.Mutex
 	*fsStat
 
+	trashMetricsLock      sync.Mutex
+	trashSpace            int64
+	trashInodes           uint64
+	delayedSlicesSpace    uint64
+	delayedSlicesCount    uint64
+
 	parentMu    sync.Mutex        // protect dirParents
 	quotaMu     sync.RWMutex      // protect dirQuotas
 	dirParents  map[Ino]Ino       // directory inode -> parent inode
@@ -268,15 +274,19 @@ type baseMeta struct {
 	prefetchMu       sync.Mutex
 	prefetchedInodes freeID
 
-	usedSpaceG   prometheus.Gauge
-	usedInodesG  prometheus.Gauge
-	totalSpaceG  prometheus.Gauge
-	totalInodesG prometheus.Gauge
-	txDist       prometheus.Histogram
-	txRestart    *prometheus.CounterVec
-	opDist       prometheus.Histogram
-	opCount      *prometheus.CounterVec
-	opDuration   *prometheus.CounterVec
+	usedSpaceG           prometheus.Gauge
+	usedInodesG          prometheus.Gauge
+	totalSpaceG          prometheus.Gauge
+	totalInodesG         prometheus.Gauge
+	trashSpaceG          prometheus.Gauge
+	trashInodesG         prometheus.Gauge
+	delayedSlicesSpaceG  prometheus.Gauge
+	delayedSlicesCountG  prometheus.Gauge
+	txDist               prometheus.Histogram
+	txRestart            *prometheus.CounterVec
+	opDist               prometheus.Histogram
+	opCount              *prometheus.CounterVec
+	opDuration           *prometheus.CounterVec
 
 	en engine
 }
@@ -322,6 +332,22 @@ func newBaseMeta(addr string, conf *Config) *baseMeta {
 			Name: "total_inodes",
 			Help: "Total number of inodes.",
 		}),
+		trashSpaceG: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "trash_space",
+			Help: "Total space used by trash in bytes.",
+		}),
+		trashInodesG: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "trash_inodes",
+			Help: "Total number of inodes in trash.",
+		}),
+		delayedSlicesSpaceG: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "delayed_slices_space",
+			Help: "Total space used by delayed slices in bytes.",
+		}),
+		delayedSlicesCountG: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "delayed_slices_count",
+			Help: "Total number of delayed slices.",
+		}),
 		txDist: prometheus.NewHistogram(prometheus.HistogramOpts{
 			Name:    "transaction_durations_histogram_seconds",
 			Help:    "Transactions latency distributions.",
@@ -355,12 +381,17 @@ func (m *baseMeta) InitMetrics(reg prometheus.Registerer) {
 	reg.MustRegister(m.usedInodesG)
 	reg.MustRegister(m.totalSpaceG)
 	reg.MustRegister(m.totalInodesG)
+	reg.MustRegister(m.trashSpaceG)
+	reg.MustRegister(m.trashInodesG)
+	reg.MustRegister(m.delayedSlicesSpaceG)
+	reg.MustRegister(m.delayedSlicesCountG)
 	reg.MustRegister(m.txDist)
 	reg.MustRegister(m.txRestart)
 	reg.MustRegister(m.opDist)
 	reg.MustRegister(m.opCount)
 	reg.MustRegister(m.opDuration)
 
+	go m.scanTrashAndDelayedSlicesMetrics()
 	go func() {
 		for {
 			if m.sessCtx != nil && m.sessCtx.Canceled() {
@@ -374,6 +405,13 @@ func (m *baseMeta) InitMetrics(reg prometheus.Registerer) {
 				m.totalSpaceG.Set(float64(totalSpace))
 				m.totalInodesG.Set(float64(iused + iavail))
 			}
+			m.trashMetricsLock.Lock()
+			m.trashSpaceG.Set(float64(m.trashSpace))
+			m.trashInodesG.Set(float64(m.trashInodes))
+			m.delayedSlicesSpaceG.Set(float64(m.delayedSlicesSpace))
+			m.delayedSlicesCountG.Set(float64(m.delayedSlicesCount))
+			m.trashMetricsLock.Unlock()
+
 			utils.SleepWithJitter(time.Second * 10)
 		}
 	}()
@@ -386,6 +424,49 @@ func (m *baseMeta) timeit(method string, start time.Time) {
 	m.opDuration.WithLabelValues(method).Add(used)
 }
 
+func (m *baseMeta) scanTrashAndDelayedSlicesMetrics() {
+	for {
+		if m.sessCtx != nil && m.sessCtx.Canceled() {
+			return
+		}
+		ctx := Background()
+
+		var trashSpace int64
+		var trashInodes uint64
+		err := m.scanTrashEntry(ctx, func(_ Ino, length uint64) {
+			trashSpace += align4K(length)
+			trashInodes++
+		})
+		if err == nil {
+			if trashSpace < 0 {
+				trashSpace = 0
+			}
+			m.trashMetricsLock.Lock()
+			m.trashSpace = trashSpace
+			m.trashInodes = trashInodes
+			m.trashMetricsLock.Unlock()
+		}
+
+		var delayedSlicesSpace uint64
+		var delayedSlicesCount uint64
+		err = m.en.scanTrashSlices(ctx, func(ss []Slice, _ int64) (bool, error) {
+			for _, s := range ss {
+				delayedSlicesSpace += uint64(s.Size)
+				delayedSlicesCount++
+			}
+			return false, nil
+		})
+		if err == nil {
+			m.trashMetricsLock.Lock()
+			m.delayedSlicesSpace = delayedSlicesSpace
+			m.delayedSlicesCount = delayedSlicesCount
+			m.trashMetricsLock.Unlock()
+		}
+
+		utils.SleepWithJitter(time.Second * 10)
+	}
+}
+
 func (m *baseMeta) getBase() *baseMeta {
 	return m
 }